Cut release 0.2.0

Signed-off-by: David Wood <[email protected]>
IBM · Jun 20, 2024 · ad398e3 · ad398e3
1 parent 4507604
commit ad398e3
Show file tree

Hide file tree

Showing 37 changed files with 73 additions and 73 deletions.
diff --git a/.make.versions b/.make.versions
@@ -11,7 +11,7 @@
 DPK_MAJOR_VERSION=0
 DPK_MINOR_VERSION=2
 DPK_MICRO_VERSION=0
-DPK_VERSION_SUFFIX=.dev6
+DPK_VERSION_SUFFIX=.dev7
 
 DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 

diff --git a/data-processing-lib/python/pyproject.toml b/data-processing-lib/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Library"

diff --git a/data-processing-lib/ray/pyproject.toml b/data-processing-lib/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_ray"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 requires-python = ">=3.10"
 description = "Data Preparation Toolkit Library for Ray"
@@ -11,7 +11,7 @@ authors = [
     { name = "Boris Lublinsky", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit==0.2.0.dev6",
+    "data-prep-toolkit==0.2.0.dev7",
     "ray[default]==2.9.3",
     # These two are to fix security issues identified by quay.io
     "fastapi>=0.109.1",

diff --git a/data-processing-lib/spark/pyproject.toml b/data-processing-lib/spark/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_spark"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 requires-python = ">=3.10"
 description = "Data Preparation Toolkit Library for Spark"
@@ -11,7 +11,7 @@ authors = [
     { name = "Constantin Adam", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit==0.2.0.dev6",	
+    "data-prep-toolkit==0.2.0.dev7",	
     "pyspark>=3.5.1",
     "pyyaml>=6.0.1",
 ]

diff --git a/kfp/kfp_ray_components/createRayClusterComponent.yaml b/kfp/kfp_ray_components/createRayClusterComponent.yaml
@@ -11,7 +11,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/deleteRayClusterComponent.yaml b/kfp/kfp_ray_components/deleteRayClusterComponent.yaml
@@ -8,7 +8,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeRayJobComponent.yaml b/kfp/kfp_ray_components/executeRayJobComponent.yaml
@@ -12,7 +12,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml b/kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml
@@ -13,7 +13,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeSubWorkflowComponent.yaml b/kfp/kfp_ray_components/executeSubWorkflowComponent.yaml
@@ -27,7 +27,7 @@ outputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists, and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_v1"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10,<3.12"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -15,7 +15,7 @@ dependencies = [
     "kfp==1.8.22",
     "ray==2.9.3",
     "requests",
-    "data_prep_toolkit==0.2.0.dev6",
+    "data_prep_toolkit==0.2.0.dev7",
 ]
 
 [build-system]

diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_v2"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10,<3.12"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}

diff --git a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_shared"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10,<3.12"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}

diff --git a/kfp/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py b/kfp/superworkflows/ray/kfp_v1/superworkflow_code_sample_wf.py
@@ -15,13 +15,13 @@
 run_fuzzy_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 run_tokenization_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 
-proglang_select_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:0.2.0.dev6"
-code_quality_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:0.2.0.dev6"
-malware_image = "quay.io/dataprep1/data-prep-kit/malware-ray:0.2.0.dev6"
-doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.0.dev6"
-ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.0.dev6"
-fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.0.dev6"
-tokenizer_image = "quay.io/dataprep1/data-prep-kit/tokenization-ray:0.2.0.dev6"
+proglang_select_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:0.2.0.dev7"
+code_quality_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:0.2.0.dev7"
+malware_image = "quay.io/dataprep1/data-prep-kit/malware-ray:0.2.0.dev7"
+doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.0.dev7"
+ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.0.dev7"
+fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.0.dev7"
+tokenizer_image = "quay.io/dataprep1/data-prep-kit/tokenization-ray:0.2.0.dev7"
 
 
 # Pipeline to invoke execution on remote resource

diff --git a/kfp/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py b/kfp/superworkflows/ray/kfp_v1/superworkflow_dedups_sample_wf.py
@@ -12,9 +12,9 @@
 run_exact_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 run_fuzzy_dedup_op = comp.load_component_from_file(component_spec_path + "executeSubWorkflowComponent.yaml")
 
-doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.0.dev6"
-ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.0.dev6"
-fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.0.dev6"
+doc_id_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.0.dev7"
+ededup_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.0.dev7"
+fdedup_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.0.dev7"
 
 # Pipeline to invoke execution on remote resource
 @dsl.pipeline(

diff --git a/transforms/code/code_quality/kfp_ray/code_quality_wf.py b/transforms/code/code_quality/kfp_ray/code_quality_wf.py
@@ -21,10 +21,10 @@
 EXEC_SCRIPT_NAME: str = "code_quality_transform_ray.py"
 PREFIX: str = ""
 
-task_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:0.2.0.dev7"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/ingest_2_parquet/kfp_ray/ingest_2_parquet_wf.py b/transforms/code/ingest_2_parquet/kfp_ray/ingest_2_parquet_wf.py
@@ -25,7 +25,7 @@
 
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/malware/kfp_ray/malware_wf.py b/transforms/code/malware/kfp_ray/malware_wf.py
@@ -21,10 +21,10 @@
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "malware_transform_ray.py"
 
-task_image = "quay.io/dataprep1/data-prep-kit/malware-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/malware-ray:0.2.0.dev7"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py b/transforms/code/proglang_select/kfp_ray/proglang_select_wf.py
@@ -21,10 +21,10 @@
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "proglang_select_transform_ray.py"
 
-task_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/proglang_select-ray:0.2.0.dev7"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/language/lang_id/kfp_ray/lang_id_multiple_wf.py b/transforms/language/lang_id/kfp_ray/lang_id_multiple_wf.py
@@ -17,13 +17,13 @@
 from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils
 
 
-task_image = "quay.io/dataprep1/data-prep-kit/lang_id-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/lang_id-ray:0.2.0.dev7"
 
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "lang_id_transform_ray.py"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/language/lang_id/kfp_ray/lang_id_wf.py b/transforms/language/lang_id/kfp_ray/lang_id_wf.py
@@ -17,13 +17,13 @@
 from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils
 
 
-task_image = "quay.io/dataprep1/data-prep-kit/lang_id-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/lang_id-ray:0.2.0.dev7"
 
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "lang_id_transform_ray.py"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/language/lang_id/python/pyproject.toml b/transforms/language/lang_id/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_lang_id_transform_python"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10"
 description = "Language Identification Python Transform"
 license = {text = "Apache-2.0"}
@@ -9,7 +9,7 @@ authors = [
     { name = "Daiki Tsuzuku", email = "[email protected]" }
 ]
 dependencies = [
-    "data-prep-toolkit==0.2.0.dev6",
+    "data-prep-toolkit==0.2.0.dev7",
     "fasttext==0.9.2",
     "langcodes==3.3.0",
     "huggingface-hub==0.21.4"

diff --git a/transforms/language/lang_id/ray/pyproject.toml b/transforms/language/lang_id/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_lang_id_transform_ray"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10"
 description = "Language Identification Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,7 +9,7 @@ authors = [
     { name = "Daiki Tsuzuku", email = "[email protected]" }
 ]
 dependencies = [
-    "dpk-lang_id-transform-python==0.2.0.dev6",
+    "dpk-lang_id-transform-python==0.2.0.dev7",
     "data-prep-toolkit-ray==0.2.0.dev6"
 ]
 

diff --git a/transforms/universal/doc_id/kfp_ray/doc_id_wf.py b/transforms/universal/doc_id/kfp_ray/doc_id_wf.py
@@ -17,12 +17,12 @@
 from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils
 
 
-task_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/doc_id-ray:0.2.0.dev7"
 
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "doc_id_transform_ray.py"
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/universal/ededup/kfp_ray/ededup_wf.py b/transforms/universal/ededup/kfp_ray/ededup_wf.py
@@ -18,13 +18,13 @@
 from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils
 
 
-task_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/ededup-ray:0.2.0.dev7"
 
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "ededup_transform_ray.py"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/universal/fdedup/kfp_ray/fdedup_wf.py b/transforms/universal/fdedup/kfp_ray/fdedup_wf.py
@@ -18,13 +18,13 @@
 from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils
 
 
-task_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/fdedup-ray:0.2.0.dev7"
 
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "fdedup_transform_ray.py"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/universal/filter/kfp_ray/filter_wf.py b/transforms/universal/filter/kfp_ray/filter_wf.py
@@ -21,10 +21,10 @@
 EXEC_SCRIPT_NAME: str = "filter_transform_ray.py"
 PREFIX: str = ""
 
-task_image = "quay.io/dataprep1/data-prep-kit/filter-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/filter-ray:0.2.0.dev7"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/universal/filter/python/pyproject.toml b/transforms/universal/filter/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_filter_transform_python"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10"
 description = "Filter Transform for Python"
 license = {text = "Apache-2.0"}
@@ -9,7 +9,7 @@ authors = [
     { name = "Constantin Adam", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit==0.2.0.dev6",
+    "data-prep-toolkit==0.2.0.dev7",
     "duckdb==0.10.1",
 ]
 

diff --git a/transforms/universal/filter/ray/pyproject.toml b/transforms/universal/filter/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_filter_transform_ray"
-version = "0.2.0.dev6"
+version = "0.2.0.dev7"
 requires-python = ">=3.10"
 description = "Filter Transform for Ray"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Constantin Adam", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-filter-transform-python==0.2.0.dev6",
-    "data-prep-toolkit-ray==0.2.0.dev6",
+    "dpk-filter-transform-python==0.2.0.dev7",
+    "data-prep-toolkit-ray==0.2.0.dev7",
 ]
 
 [build-system]

diff --git a/transforms/universal/noop/kfp_ray/noop_multiple_wf.py b/transforms/universal/noop/kfp_ray/noop_multiple_wf.py
@@ -17,13 +17,13 @@
 from workflow_support.compile_utils import ONE_HOUR_SEC, ONE_WEEK_SEC, ComponentUtils
 
 
-task_image = "quay.io/dataprep1/data-prep-kit/noop-ray:0.2.0.dev6"
+task_image = "quay.io/dataprep1/data-prep-kit/noop-ray:0.2.0.dev7"
 
 # the name of the job script
 EXEC_SCRIPT_NAME: str = "noop_transform_ray.py"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev6"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.0.dev7"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"