Fuzzy dedup

data-processing-lib/spark/Makefile

@@ -11,9 +11,14 @@ setup::
 
 set-versions: .check-env
 	$(MAKE) TOML_VERSION=$(DPK_LIB_VERSION) .defaults.update-toml
-	sed -e 's/"pyspark...*",/"pyspark>=${SPARK_VERSION}",/'				\
-	    pyproject.toml > tt.toml
-	mv tt.toml pyproject.toml
+	if [ -e pyproject.toml ]; then					\
+		cat pyproject.toml | sed -e 's/"spark[default]==.*",/"spark[default]==$(SPARK_VERSION)",/' > tt.toml; \
+		mv tt.toml pyproject.toml; \
+	fi
+	if [ -e requirements.txt ]; then					\
+		cat requirements.txt | sed -e 's/ray[default]==.*/ray[default]==$(SPARK_VERSION)/' > tt.txt; \
+		mv tt.txt requirements.txt; \
+	fi
 
 build:: build-dist 
 
@@ -26,7 +31,7 @@ publish-dist :: .check-env .defaults.publish-dist
 
 publish-image:: .defaults.publish-image
 
-venv::  pyproject.toml
+venv::
 	$(MAKE) .defaults.spark-lib-src-venv
 	pip install pytest pytest-cov 
 

scripts/check-workflows.sh

@@ -17,7 +17,7 @@ if [ ! -d transforms ]; then
     echo Please run this script from the top of the repository
     exit 1
 fi
-KFP_BLACK_LIST="doc_chunk pdf2parquet pii_redactor text_encoder license_select repo_level_ordering header_cleanser"
+KFP_BLACK_LIST="doc_chunk pdf2parquet pii_redactor text_encoder license_select repo_level_ordering header_cleanser fdedup"
 while [ $# -ne 0 ]; do
    case $1 in
         -show-kfp-black-list)    echo $KFP_BLACK_LIST; exit 0;

transforms/universal/fdedup/README.md

@@ -1,10 +1,11 @@
-# Fuzzy Deduplification Transform 
-The fdedup transforms removes documents that are very similar to each other within a set of parquet files, 
-per the set of 
-[transform project conventions](../../README.md#transform-project-conventions)
-the following runtimes are available:
+# Fuzzy Deduplication Transform 
+The fdedup transform eliminates documents that are highly similar to each other (but not necessarily identical) from a
+set of Parquet files. This ensures that the resulting dataset contains only unique or sufficiently distinct entries.
+Per the set of [transform project conventions](../../README.md#transform-project-conventions) the following runtimes are available:
 
-* [ray](ray/README.md) - enables the running of the base python transformation
-in a Ray runtime
-* [kfp](kfp_ray/README.md) - enables running the ray docker image 
-in a kubernetes cluster using a generated `yaml` file.
+* [python](python/README.md) - enables running the base transform in a pure python environment
+* [ray](ray/README.md) - enables running the base python transform in a Ray runtime
+* [spark](spark/README.md) - enables running the base python transform in a spark runtime
+* [kfp](kfp_ray/README.md) - enables running the ray docker image in a kubernetes cluster using a generated `yaml` file.
+
+Please check [here](python/README.md) for a more detailed description of this transform.

transforms/universal/fdedup/kfp_ray/README.md

@@ -1,8 +1,8 @@
-# Fuzzy Deduplication Ray-base KubeFlow Pipeline Transformation 
+# Fuzzy Deduplication Ray-based KubeFlow Pipeline Transformation 
 
 
 ## Summary 
-This project allows execution of the [noop Ray transform](../ray) as a 
+This project allows execution of the [fuzzy dedup Ray transform](../ray) as a 
 [KubeFlow Pipeline](https://www.kubeflow.org/docs/components/pipelines/overview/)
 
 The detail pipeline is presented in the [Simplest Transform pipeline tutorial](../../../../kfp/doc/simple_transform_pipeline.md) 
@@ -16,13 +16,9 @@ make workflow-build
 from the directory. It creates a virtual environment (make workflow-venv) and after that compiles the pipeline 
 definitions in the folder. The virtual environment is created once for all transformers. 
 
-Note: the pipelines definitions can be compiled and executed on KFPv1 and KFPv2. Meantime, KFPv1 is our default. If you
-prefer KFPv2, please do the following:
-```shell
-make clean
-export KFPv2=1
-make workflow-build
-```
+## Considerations
+Currently, fuzzy dedup KFP pipeline definitions can be compiled and executed on KFPv1. KFPv2 is not
+supported currently, because of this issue: https://github.com/kubeflow/pipelines/issues/10914
 
 The next steps are described in [Deploying a pipeline](../../../../kfp/doc/simple_transform_pipeline.md#deploying-a-pipeline-)
 and [Executing pipeline and watching execution results](../../../../kfp/doc/simple_transform_pipeline.md#executing-pipeline-and-watching-execution-results-)