build transforms==0.2.3.dev0 release with toolkit==0.2.2

Signed-off-by: Maroun Touma <[email protected]>
IBM · Nov 26, 2024 · dae2276 · dae2276
1 parent 2f80d9c
commit dae2276
Show file tree

Hide file tree

Showing 120 changed files with 938 additions and 202 deletions.
diff --git a/.make.versions b/.make.versions
@@ -16,10 +16,10 @@ DPK_MAJOR_VERSION=0
 # The minor version is incremented manually when significant features have been added that are backward compatible with the previous major.minor release.
 DPK_MINOR_VERSION=2
 # The minor version is incremented AUTOMATICALLY by the release.sh script when a new release is set.
-DPK_MICRO_VERSION=3
+DPK_MICRO_VERSION=2
 # The suffix is generally always set in the main/development branch and only nulled out when creating release branches.
 # It can be manually incremented, for example, to allow publishing a new intermediate version wheel to pypi. 
-DPK_VERSION_SUFFIX=.dev0
+DPK_VERSION_SUFFIX=
 
 DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 

diff --git a/data-processing-lib/pyproject.toml b/data-processing-lib/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.2.3.dev0"
+version = "0.2.2"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Ray and Python"

diff --git a/kfp/kfp_ray_components/createRayClusterComponent.yaml b/kfp/kfp_ray_components/createRayClusterComponent.yaml
@@ -11,7 +11,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/deleteRayClusterComponent.yaml b/kfp/kfp_ray_components/deleteRayClusterComponent.yaml
@@ -9,7 +9,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeRayJobComponent.yaml b/kfp/kfp_ray_components/executeRayJobComponent.yaml
@@ -12,7 +12,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml b/kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml
@@ -13,7 +13,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeSubWorkflowComponent.yaml b/kfp/kfp_ray_components/executeSubWorkflowComponent.yaml
@@ -27,7 +27,7 @@ outputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists, and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_v1"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -13,7 +13,7 @@ authors = [
 ]
 dependencies = [
     "kfp==1.8.22",
-    "data-prep-toolkit-kfp-shared==0.2.3.dev0",
+    "data-prep-toolkit-kfp-shared==0.2.2",
 ]
 
 [build-system]

diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_v2"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -14,7 +14,7 @@ authors = [
 dependencies = [
     "kfp==2.8.0",
     "kfp-kubernetes==1.2.0",
-    "data-prep-toolkit-kfp-shared==0.2.3.dev0",
+    "data-prep-toolkit-kfp-shared==0.2.2",
 ]
 
 [build-system]

diff --git a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_shared"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -14,7 +14,7 @@ authors = [
 dependencies = [
     "requests",
     "kubernetes",
-    "data-prep-toolkit[ray]==0.2.3.dev0",
+    "data-prep-toolkit[ray]>=0.2.2",
 ]
 
 [build-system]

diff --git a/transforms/Makefile b/transforms/Makefile
@@ -107,7 +107,7 @@ build-pkg-dist:
 	-rm -fr src
 	mkdir src
 	# Copy all the src folders recursively (not clear if they have subfolders)
-	for x in $(shell find . | grep '[ray| python]/src$$') ; do \
+	for x in $(shell find . | grep '[ray| python | spark]/src$$') ; do \
 		echo $$x ; \
 		if [ -d "$$x" ]; then \
 		   cp -r $$x/* src ; \

diff --git a/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py b/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py
@@ -25,7 +25,7 @@
 
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/code2parquet/python/pyproject.toml b/transforms/code/code2parquet/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code2parquet_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "code2parquet Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/code2parquet/python/requirements.txt b/transforms/code/code2parquet/python/requirements.txt
@@ -1,3 +1,3 @@
-data-prep-toolkit==0.2.3.dev0
+data-prep-toolkit>=0.2.2
 parameterized
 pandas
diff --git a/transforms/code/code2parquet/ray/pyproject.toml b/transforms/code/code2parquet/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code2parquet_transform_ray"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "code2parquet Ray Transform"
 license = {text = "Apache-2.0"}
@@ -10,8 +10,8 @@ authors = [
     { name = "Boris Lublinsky", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit[ray]==0.2.3.dev0",
-    "dpk-code2parquet-transform-python==0.2.3.dev0",
+    "data-prep-toolkit[ray]>=0.2.2",
+    "dpk-code2parquet-transform-python==0.2.2",
     "parameterized",
     "pandas",
 ]

diff --git a/transforms/code/code_profiler/python/pyproject.toml b/transforms/code/code_profiler/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_profiler_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Code Profiler Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/code_profiler/python/requirements.txt b/transforms/code/code_profiler/python/requirements.txt
@@ -1,4 +1,4 @@
-data-prep-toolkit==0.2.3.dev0
+data-prep-toolkit>=0.2.2
 parameterized
 pandas
 aiolimiter==1.1.0

diff --git a/transforms/code/code_profiler/ray/pyproject.toml b/transforms/code/code_profiler/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_profiler_transform_ray"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Code Profiler Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Pankaj Thorat", email = "[email protected]" },
 ]
 dependencies = [
-	"dpk-code-profiler-transform-python==0.2.3.dev0",
-    "data-prep-toolkit[ray]==0.2.3.dev0",
+	"dpk-code-profiler-transform-python==0.2.2",
+    "data-prep-toolkit[ray]>=0.2.2",
 	]
 
 [build-system]

diff --git a/transforms/code/code_quality/kfp_ray/code_quality_wf.py b/transforms/code/code_quality/kfp_ray/code_quality_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/code_quality/python/pyproject.toml b/transforms/code/code_quality/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_quality_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Code Quality Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/code_quality/python/requirements.txt b/transforms/code/code_quality/python/requirements.txt
@@ -1,3 +1,3 @@
-data-prep-toolkit==0.2.3.dev0
+data-prep-toolkit>=0.2.2
 bs4==0.0.2
 transformers==4.38.2
diff --git a/transforms/code/code_quality/ray/pyproject.toml b/transforms/code/code_quality/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_quality_transform_ray"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Code Quality Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Shivdeep Singh", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-code-quality-transform-python==0.2.3.dev0",
-    "data-prep-toolkit[ray]==0.2.3.dev0",
+    "dpk-code-quality-transform-python==0.2.2",
+    "data-prep-toolkit[ray]>=0.2.2",
 ]
 
 [build-system]

diff --git a/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py b/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/header_cleanser-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/header_cleanser/python/pyproject.toml b/transforms/code/header_cleanser/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_header_cleanser_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "License and Copyright Removal Transform for Python"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/header_cleanser/python/requirements.txt b/transforms/code/header_cleanser/python/requirements.txt
@@ -1,3 +1,3 @@
-data-prep-toolkit==0.2.3.dev0
+data-prep-toolkit>=0.2.2
 scancode-toolkit==32.1.0 ; platform_system != 'Darwin'
 
diff --git a/transforms/code/header_cleanser/ray/pyproject.toml b/transforms/code/header_cleanser/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_header_cleanser_transform_ray"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "License and copyright removal Transform for Ray"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Yash kalathiya", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-header-cleanser-transform-python==0.2.3.dev0",
-    "data-prep-toolkit[ray]==0.2.3.dev0",
+    "dpk-header-cleanser-transform-python==0.2.2",
+    "data-prep-toolkit[ray]>=0.2.2",
     "scancode-toolkit==32.1.0",
 ]
 

diff --git a/transforms/code/license_select/kfp_ray/license_select_wf.py b/transforms/code/license_select/kfp_ray/license_select_wf.py
@@ -25,7 +25,7 @@
 
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/license_select/python/pyproject.toml b/transforms/code/license_select/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_license_select_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "License Select Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/license_select/python/requirements.txt b/transforms/code/license_select/python/requirements.txt
@@ -1 +1 @@
-data-prep-toolkit==0.2.3.dev0
+data-prep-toolkit>=0.2.2
diff --git a/transforms/code/license_select/ray/pyproject.toml b/transforms/code/license_select/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_license_select_transform_ray"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "License Select Transform"
 license = {text = "Apache-2.0"}
@@ -10,8 +10,8 @@ authors = [
     { name = "Mark Lewis", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-license-select-transform-python==0.2.3.dev0",
-    "data-prep-toolkit[ray]==0.2.3.dev0",
+    "dpk-license-select-transform-python==0.2.2",
+    "data-prep-toolkit[ray]>=0.2.2",
 ]
 
 [build-system]

diff --git a/transforms/code/malware/kfp_ray/malware_wf.py b/transforms/code/malware/kfp_ray/malware_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/malware-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/malware/python/pyproject.toml b/transforms/code/malware/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_malware_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Malware Python Transform"
 license = {text = "Apache-2.0"}
@@ -9,7 +9,7 @@ authors = [
     { name = "Takuya Goto", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit==0.2.3.dev0",
+    "data-prep-toolkit>=0.2.2",
     "clamd==1.0.2",
 ]
 

diff --git a/transforms/code/malware/ray/pyproject.toml b/transforms/code/malware/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_malware_transform_ray"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Malware Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Takuya Goto", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-malware-transform-python==0.2.3.dev0",
-    "data-prep-toolkit[ray]==0.2.3.dev0",
+    "dpk-malware-transform-python==0.2.2",
+    "data-prep-toolkit[ray]>=0.2.2",
 ]
 
 [build-system]

diff --git a/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py b/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.2"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/proglang_select/python/pyproject.toml b/transforms/code/proglang_select/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_proglang_select_transform_python"
-version = "0.2.3.dev0"
+version = "0.2.2"
 requires-python = ">=3.10,<3.13"
 description = "Programming Language Selection Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/proglang_select/python/requirements.txt b/transforms/code/proglang_select/python/requirements.txt
@@ -1 +1 @@
-data-prep-toolkit==0.2.3.dev0
+data-prep-toolkit>=0.2.2
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		data-prep-toolkit==0.2.3.dev0
		data-prep-toolkit>=0.2.2