Build transforms wheel

transforms/packaging/.gitignore

@@ -0,0 +1,5 @@
+**/src
+**/dist
+**/*.egg-info
+**/build
+

transforms/packaging/.make.packaging

@@ -0,0 +1,48 @@
+
+venv:  
+	$(MAKE) .defaults.create-venv
+
+test::	setup test-src
+	@# Help: Setup environment, load wheel from pyproject.toml and run unit tests for all transforms
+
+clean:: .transforms.clean
+	-rm -fr src
+
+image:: .transforms.python-image
+
+test-src:: 
+	source venv/bin/activate;       \
+	for T in $(TRANSFORMS_NAMES); do                    \
+	    echo running unit test on: $$T ; \
+		$(PYTEST) $(REPOROOT)/transforms/$$T/$(PACKAGING_RUN_TIME)/test; \
+	done;
+	@# Help: Run all unit tests from the same venv environment (should follow make venv)
+
+
+setup: .transforms.setup venv
+	$(MAKE) src
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install . 
+	@# Help: Do any default transform setup before running make src
+
+
+src:
+	for T in $(TRANSFORMS_NAMES); do                    \
+	    echo copy src from  $$T ; \
+		cp -R $(REPOROOT)/transforms/$$T/$(PACKAGING_RUN_TIME)/src/ src/ ; \
+		rm -fr *.egg-info ; \
+		rm -fr dist ; \
+		rm -fr build ; \
+	done;
+	@# Help: Setup src folder and remove old distribution
+
+
+build:: build-dist
+
+publish:: publish-dist
+
+build-dist:: .defaults.build-dist 
+
+publish-dist:: .defaults.publish-dist
+
+

transforms/packaging/python/Makefile

@@ -0,0 +1,42 @@
+# Define the root of the local git clone for the common rules to be able
+# know where they are running from.
+REPOROOT=../../..
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below.
+
+# $(REPOROOT)/.make.versions file contains the versions
+
+include $(REPOROOT)/transforms/.make.transforms
+include ../.make.packaging
+
+PACKAGING_RUN_TIME=python
+DPK_TRNASFORM_REV=0.2.1.dev1
+
+TRANSFORMS_NAMES = code/code_quality \
+    	code/code2parquet \
+    	code/header_cleanser \
+		code/code_quality \
+		code/proglang_select \
+		language/doc_chunk \
+		language/doc_quality \
+		language/lang_id \
+		language/pdf2parquet \
+		language/text_encoder \
+        language/pii_redactor \
+		universal/ededup \
+		universal/filter \
+		universal/resize \
+		universal/tokenization \
+        universal/html2parquet
+
+test-with-pypi:
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install data_prep_toolkit_transforms==$(DPK_TRNASFORM_REV) 
+	$(MAKE) test-src
+	@# Help: Load wheel from pypi and run all unit tests: final step in verification after deploying to pypi)
+
+
+
+

transforms/packaging/python/README.md

@@ -0,0 +1,33 @@
+# DPK Python Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms`
+
+installing the python transforms will also install  `data-prep-toolkit`
+
+## List of Transforms in current package
+
+* code
+    * [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/python/README.md)
+    * header_cleanser (Not available on MacOS)
+    * code_quality
+    * proglang_select
+* language
+    * doc_chunk
+	* *doc_quality
+	* lang_id
+	* pdf2parquet
+	* text_encoder
+* universal
+    * ededup
+	* filter
+	* resize
+	* tokenization
+
+
+
+
+

transforms/packaging/python/pyproject.toml

@@ -0,0 +1,63 @@
+[project]
+name = "data_prep_toolkit_transforms"
+version = "0.2.1.dev1"
+requires-python = ">=3.10,<3.12"
+keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
+description = "Data Preparation Toolkit Transforms"
+license = {text = "Apache-2.0"}
+readme = {file = "README.md", content-type = "text/markdown"}
+authors = [
+    { name = "Maroun Touma", email = "[email protected]" },
+]
+
+dependencies = [
+	"data-prep-toolkit==0.2.1.dev1",
+	"argparse",
+	"boto3==1.34.69",
+	"bs4==0.0.2",
+	"clamd==1.0.2",
+	"docling[ocr]==1.1.2",
+	"duckdb==0.10.1",
+	"fasttext==0.9.2",
+	"filetype >=1.2.0, <2.0.0",
+	"huggingface-hub >= 0.21.4, <1.0.0",
+	"langcodes==3.3.0",
+	"mmh3==4.1.0",
+	"numpy==1.26.4",
+	"pandas",
+	"parameterized",
+	"pyarrow==16.1.0",
+	"python-dateutil>=2.8.2",
+	"pytz>=2020.1",
+	"quackling==0.1.0",
+	"scancode-toolkit==32.1.0 ; platform_system != 'Darwin'",
+	"sentence-transformers==3.0.1",
+	"transformers==4.38.2",
+	"tzdata>=2022.7",
+	"xxhash==3.4.1",
+]
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+
+
+[options]
+package_dir = ["src"]
+
+[options.packages.find]
+where = ["src/"]
+
+[tool.pytest.ini_options]
+# Currently we use low coverage since we have to run tests separately (see makefile)
+#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
+markers = ["unit: unit tests", "integration: integration tests"]
+
+[tool.coverage.run]
+include = ["src/*"]
+
+
+
+
+
+

transforms/packaging/ray/Makefile

@@ -0,0 +1,53 @@
+# Define the root of the local git clone for the common rules to be able
+# know where they are running from.
+REPOROOT=../../..
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below.
+
+# $(REPOROOT)/.make.versions file contains the versions
+
+include $(REPOROOT)/transforms/.make.transforms
+include ../.make.packaging
+
+PACKAGING_RUN_TIME=ray
+DPK_TRNASFORM_REV=0.2.1.dev0
+
+## Ray Transforms:  `find . -name src | grep ray/src`
+TRANSFORMS_NAMES = code/proglang_select \
+	code/header_cleanser \
+	code/code_quality \
+	code/repo_level_ordering \
+	code/code2parquet \
+	language/doc_quality \
+	language/doc_chunk \
+	language/lang_id \
+	language/text_encoder \
+	language/pdf2parquet \
+	language/pii_redactor \
+	universal/fdedup \
+	universal/tokenization \
+	universal/ededup \
+	universal/profiler \
+	universal/doc_id \
+	universal/filter \
+	universal/resize
+
+test-with-local-python:
+	$(MAKE) clean
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+	cd ../python; \
+	$(PYTHON) -m pip install . ; \
+	cd ../ray; \
+	$(PYTHON) -m pip install . ; \
+	$(PYTHON) -m pip install data_prep_toolkit_transforms_ray==$(DPK_TRNASFORM_REV)
+	$(MAKE) test-src
+
+test-with-pypi:
+	$(MAKE) clean
+	$(MAKE) .defaults.create-venv
+	source venv/bin/activate;       \
+	$(PYTHON) -m pip install data_prep_toolkit_transforms_ray==$(DPK_TRNASFORM_REV) 
+	$(MAKE) test-src
+

transforms/packaging/ray/README.md

@@ -0,0 +1,37 @@
+# DPK Ray Transforms
+
+## installation
+
+The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
+
+`python -m pip install data-prep-toolkit-transforms-ray`
+
+installing the Ray transforms will also install `data_prep_toolkit_transforms` and `data-prep-toolkit-ray`
+
+## List of Ray Transforms availabe in current package
+
+* code
+	* [code2parquet](https://github.com/IBM/data-prep-kit/blob/dev/transforms/code/code2parquet/ray/README.md)
+	* proglang_select
+	* header_cleanser (Not available on MacOS)
+	* code_quality
+	* repo_level_ordering
+* language
+	* doc_quality
+	* doc_chunk
+	* lang_id
+	* text_encoder
+	* pdf2parquet
+* universal
+	* fdedup
+	* tokenization
+	* ededup
+	* profiler
+	* doc_id
+	* filter
+	* resize
+
+
+
+
+

transforms/packaging/ray/pyproject.toml

@@ -0,0 +1,53 @@
+[project]
+name = "data_prep_toolkit_transforms_ray"
+version = "0.2.1.dev1"
+requires-python = ">=3.10,<3.12"
+keywords = ["transforms", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
+description = "Data Preparation Toolkit Transforms using Ray"
+license = {text = "Apache-2.0"}
+readme = {file = "README.md", content-type = "text/markdown"}
+authors = [
+    { name = "Maroun Touma", email = "[email protected]" },
+]
+
+dependencies = [
+	#"data_prep_toolkit_transforms==0.2.1.dev1",
+	"data-prep-toolkit-ray==0.2.1.dev0",
+	"scancode-toolkit==32.1.0 ; platform_system != 'Darwin'",
+	"parameterized",
+	"tqdm==4.66.3",
+	"mmh3==4.1.0",
+	"xxhash==3.4.1",
+	"tqdm==4.66.3",
+	"scipy==1.12.0",
+	"networkx==3.3",
+	"colorlog==6.8.2",
+	"func-timeout==4.3.5",
+	"pandas==2.2.2",
+	"emerge-viz==2.0.0",
+]
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+
+
+[options]
+package_dir = ["src"]
+
+[options.packages.find]
+where = ["src/"]
+
+[tool.pytest.ini_options]
+# Currently we use low coverage since we have to run tests separately (see makefile)
+#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
+markers = ["unit: unit tests", "integration: integration tests"]
+
+[tool.coverage.run]
+include = ["src/*"]
+
+
+
+
+
+