merge with dev with latest update for readme.md and notebook

Signed-off-by: Maroun Touma <[email protected]>
IBM · Nov 24, 2024 · 3c0187c · 3c0187c
2 parents 6cda6fb + 5a018e6
commit 3c0187c
Show file tree

Hide file tree

Showing 42 changed files with 1,029 additions and 85 deletions.
diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/code2parquet/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/code2parquet/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/code_quality/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/code_quality/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/header_cleanser/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/header_cleanser/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/license_select/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/license_select/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/malware/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/malware/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/proglang_select/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/proglang_select/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/repo_level_ordering/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/code/repo_level_ordering/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "@TARGET_TRANSFORM_DIR@/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "@TARGET_TRANSFORM_DIR@/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-doc_chunk-kfp.yml b/.github/workflows/test-language-doc_chunk-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/doc_chunk/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/doc_chunk/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/doc_quality/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/doc_quality/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-html2parquet-kfp.yml b/.github/workflows/test-language-html2parquet-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/html2parquet/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/html2parquet/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/lang_id/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/lang_id/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-pdf2parquet-kfp.yml b/.github/workflows/test-language-pdf2parquet-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/pdf2parquet/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/pdf2parquet/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-pii_redactor-kfp.yml b/.github/workflows/test-language-pii_redactor-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/pii_redactor/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/pii_redactor/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/text_encoder/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/language/text_encoder/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/doc_id/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/doc_id/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/ededup/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/ededup/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/fdedup/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/fdedup/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/filter/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/filter/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-hap-kfp.yml b/.github/workflows/test-universal-hap-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/hap/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/hap/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/noop/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/noop/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/profiler/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/profiler/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/resize/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/resize/**"
             - "!data-processing-lib/**" # This is tested in separate workflow

diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml
@@ -13,6 +13,7 @@ on:
             - "*"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/tokenization/**"
             - "!kfp/**" # This is tested in separate workflow
@@ -27,6 +28,7 @@ on:
             - "releases/**"
         paths:
             - ".make.*"
+            - "scripts/k8s-setup/requirements.env"
             - "transforms/.make.workflows"
             - "transforms/universal/tokenization/**"
             - "!data-processing-lib/**" # This is tested in separate workflow