Build transforms wheel

.make.versions

@@ -19,7 +19,7 @@ DPK_MINOR_VERSION=2
 DPK_MICRO_VERSION=1
 # The suffix is generally always set in the main/development branch and only nulled out when creating release branches.
 # It can be manually incremented, for example, to allow publishing a new intermediate version wheel to pypi. 
-DPK_VERSION_SUFFIX=.dev0
+DPK_VERSION_SUFFIX=.dev3
 
 DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 
@@ -103,6 +103,8 @@ PII_REDACTOR_PYTHON_VERSION=$(DPK_VERSION)
 
 HTML2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
 
+DPK_TRANSFORMS_VERSION=$(DPK_VERSION)
+
 ################## ################## ################## ################## ################## ##################
 # Begin versions that the repo depends on. 
 
@@ -117,3 +119,4 @@ ifeq ($(KFPv2), 1)
 else
         WORKFLOW_SUPPORT_LIB=kfp_v1_workflow_support
 endif
+

data-processing-lib/python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.2.1.dev0"
+version = "0.2.1.dev3"
 requires-python = ">=3.10"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 description = "Data Preparation Toolkit Library"

data-processing-lib/ray/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_ray"
-version = "0.2.1.dev0"
+version = "0.2.1.dev3"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 requires-python = ">=3.10"
 description = "Data Preparation Toolkit Library for Ray"
@@ -11,7 +11,7 @@ authors = [
     { name = "Boris Lublinsky", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit==0.2.1.dev0",
+    "data-prep-toolkit>=0.2.1.dev3",
     "ray[default]==2.24.0",
     # These two are to fix security issues identified by quay.io
     "fastapi>=0.110.2",

examples/notebooks/rag/requirements.txt

@@ -1,6 +1,6 @@
 ## Data prep kit
-data-prep-toolkit-transforms==0.2.1.dev1
-data-prep-toolkit-transforms-ray==0.2.1.dev1
+#data-prep-toolkit-transforms==0.2.1.dev1
+#data-prep-toolkit-transforms-ray==0.2.1.dev1
 
 
 
@@ -53,4 +53,4 @@ ipython
 ipywidgets
 IProgress
 chardet==5.2.0
-charset-normalizer==3.3.2
+charset-normalizer==3.3.2

transforms/packaging/.gitignore

@@ -0,0 +1,5 @@
+**/src
+**/dist
+**/*.egg-info
+**/build
+

transforms/packaging/.make.packaging

@@ -0,0 +1,48 @@
+
+venv:  
+	$(MAKE) .defaults.create-venv
+
+test::	test-src
+
+clean:: .transforms.clean
+	-rm -fr src
+
+image:: .transforms.python-image
+
+run-ut::
+	source venv/bin/activate;       \
+	for T in $(TRANSFORMS_NAMES); do                    \
+	    echo running unit test on: $$T ; \
+		$(PYTEST) $(REPOROOT)/transforms/$$T/$(PACKAGING_RUN_TIME)/test; \
+	done;
+	@# Help: Setup environment and run unit tests for all transforms
+
+
+setup: .transforms.setup venv
+	$(MAKE) src
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install . 
+	@# Help: Do any default transform setup before running make src and setting up a test environment
+
+
+src:
+	mkdir src
+	for T in $(TRANSFORMS_NAMES); do                    \
+	    echo copy src from  $$T ; \
+		cp -R $(REPOROOT)/transforms/$$T/$(PACKAGING_RUN_TIME)/src/* src ; \
+		rm -fr *.egg-info ; \
+		rm -fr dist ; \
+		rm -fr build ; \
+	done;
+	@# Help: Setup src folder and remove old distribution
+
+
+build:: build-dist
+
+publish:: publish-dist
+
+build-dist:: src .defaults.build-dist 
+
+publish-dist:: .defaults.publish-dist
+
+

transforms/packaging/Makefile

@@ -0,0 +1,48 @@
+REPOROOT=../../
+# Use make help, to see the available rules
+include ../../.make.defaults
+
+setup::
+
+clean::
+	# Clean up workflows common virtual environment.
+	rm -rf venv || true
+	rm -rf *.back || true
+	@# Help: Recursively make $@ all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+setup::
+
+build::
+
+venv::
+
+image:: 
+
+publish:: 
+
+test-image:: 
+
+test:: 
+
+test-src::
+	@# Help: Recursively make $@ in all subdirs 
+	$(MAKE) RULE=$@ .recurse
+
+kind-load-image::
+
+docker-load-image::
+
+docker-save-image::
+
+workflow-venv::
+
+workflow-test::
+
+workflow-build::
+
+workflow-upload::
+
+set-versions::
+	@# Help: Recursively make $@ in all subdirs
+	@$(MAKE) RULE=$@ .recurse

transforms/packaging/python/Makefile

@@ -0,0 +1,61 @@
+# Define the root of the local git clone for the common rules to be able
+# know where they are running from.
+REPOROOT=../../..
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below.
+
+# $(REPOROOT)/.make.versions file contains the versions
+
+include $(REPOROOT)/transforms/.make.transforms
+include ../.make.packaging
+
+PACKAGING_RUN_TIME=python
+
+
+#Excluded List
+# ./code/malware
+# ./universal/html2parquet
+# ./universal/profiler      # Missing implementation 
+# ./universal/fdedup 	    # Missing implementation
+#  code/repo_level_ordering # Missing implementation
+
+
+TRANSFORMS_NAMES = code/code_quality \
+	    	code/code2parquet \
+    		code/header_cleanser \
+		code/proglang_select \
+		language/doc_chunk \
+		language/doc_quality \
+		language/lang_id \
+		language/pdf2parquet \
+		language/pii_redactor \
+		language/text_encoder \
+		universal/ededup \
+		universal/filter \
+		universal/resize \
+		universal/tokenization \
+		universal/doc_id
+
+
+# distribution versions is the same as image version.
+set-versions:
+	$(MAKE) TRANSFORM_PYTHON_VERSION=$(DPK_TRANSFORMS_VERSION) TOML_VERSION=$(DPK_TRANSFORMS_VERSION) .transforms.set-versions 
+
+test-src:: .transforms.setup venv
+	source venv/bin/activate && $(PYTHON) -m pip install ../../../data-processing-lib/python
+	source venv/bin/activate && $(MAKE) src && $(PYTHON) -m pip install .
+	$(MAKE) run-ut
+	@# Help: Do any default transform setup before running make src and setting up a test environment
+
+test-with-pypi:
+	$(MAKE) clean
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install data_prep_toolkit_transforms==$(DPK_TRANSFORMS_VERSION) 
+	$(MAKE) run-ut
+	@# Help: Load wheel from pypi and run all unit tests: final step in verification after deploying to pypi)
+
+
+
+

transforms/packaging/python/README.md

@@ -0,0 +1,37 @@
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * [header_cleanser (Not available on MacOS)](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/header_cleanser/python/README.md)
+    * [code_quality](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code_quality/python/README.md)
+    * [proglang_select](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/proglang_select/python/README.md)
+* language
+    * [doc_chunk](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_chunk/python/README.md)
+	* [doc_quality](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_quality/python/README.md)
+	* [lang_id](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/lang_id/python/README.md)
+	* [pdf2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/pdf2parquet/python/README.md)
+	* [text_encoder](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/text_encoder/python/README.md)
+	* [pii_redactor](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/pii_redactor/python/README.md)
+* universal
+    * [ededup](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/ededup/python/README.md)
+	* [filter](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/filter/python/README.md)
+	* [resize](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/resize/python/README.md)
+	* [tokenization](https://github.com/IBM/data-prep-kit/blob/dev/transforms/tokenization/doc_chunk/python/README.md)
+	* [doc_id](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/doc_id/python/README.md)
+
+
+
+
+
+
+

transforms/packaging/python/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "data_prep_toolkit_transforms"
+version = "0.2.1.dev3"
+requires-python = ">=3.10,<3.12"
+keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
+description = "Data Preparation Toolkit Transforms"
+license = {text = "Apache-2.0"}
+readme = {file = "README.md", content-type = "text/markdown"}
+authors = [
+    { name = "Maroun Touma", email = "[email protected]" },
+]
+dynamic = ["dependencies"]
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.transforms.python.txt"]}
+
+[options]
+package_dir = ["src"]
+
+[options.packages.find]
+where = ["src/"]
+
+[tool.pytest.ini_options]
+# Currently we use low coverage since we have to run tests separately (see makefile)
+#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
+markers = ["unit: unit tests", "integration: integration tests"]
+
+[tool.coverage.run]
+include = ["src/*"]
+
+
+
+
+
+

transforms/packaging/python/requirements.transforms.python.txt

@@ -0,0 +1,39 @@
+data-prep-toolkit>=0.2.1.dev3
+bs4==0.0.2
+#docling 1.9.0 depends on docling-parse<2.0.0 and >=1.1.3
+#pdf2parquet depends on docling-parse==1.0.0
+#docling 1.8.5 depends on docling-parse<2.0.0 and >=1.1.3
+#docling-parse>=1.0.0,
+# language/doc_chunk has conflict dependencies with pdf2parquet that need to be resolved
+# doc_chunk depends on docling>=1.8.2,<2.0.0
+# pdf2parquet depends on docling==1.7.0
+#docling==1.7.0,
+#pdf2parquet
+docling-core==1.2.0,
+docling==1.11.0,
+filetype >=1.2.0, <2.0.0,
+#DockChunking
+quackling==0.4.0,
+# quackling will pull 
+#  docling>=1.8.2,<2.0.0 
+#  llama-index-core<0.12.0,>=0.11.1
+#  docling-core<2.0.0,>=1.1.2
+duckdb==0.10.1
+fasttext==0.9.2
+huggingface-hub >= 0.21.4, <1.0.0
+langcodes==3.3.0
+mmh3==4.1.0
+numpy==1.26.4
+pandas
+parameterized
+scancode-toolkit==32.1.0 ; platform_system != 'Darwin'
+sentence-transformers==3.0.1
+transformers==4.38.2
+xxhash==3.4.1
+# PII-redactor
+presidio-analyzer>=2.2.355
+presidio-anonymizer>=2.2.355
+flair>=0.14.0
+pandas>=2.2.2
+
+