automatically determine transforms in transforms directory for which …

…to generate test workflows

Signed-off-by: David Wood <[email protected]>

.github/workflows/Makefile

@@ -8,13 +8,19 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco
 
 
 transform-tests:
-	for i in $(CODE_TRANSFORMS); do	\
-	    cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/code/$$i?g" > test-code-$$i.yml;	\
-	done
-	for i in $(LANG_TRANSFORMS); do	\
-	    cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/language/$$i?g" > test-language-$$i.yml;	\
-	done
-	for i in $(UNIVERSAL_TRANSFORMS); do	\
-	    cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/universal/$$i?g" > test-universal-$$i.yml;	\
+	$(MAKE) TRANSFORM_SUBDIR=universal .transform-tests
+	$(MAKE) TRANSFORM_SUBDIR=language .transform-tests
+	$(MAKE) TRANSFORM_SUBDIR=code .transform-tests
+
+# Expects
+#   TRANSFORM_SUBDIR  transforms subdirectory (such as universal)
+.transform-tests:
+	@for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR)  -depth 1 -type d); do	\
+	    dir=$$(basename $$i);							\
+	    yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml;					\
+	    echo Generating $$yml;							\
+	    cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;	\
 	done
 
+
+

.github/workflows/README.md

@@ -1,6 +1,6 @@
 # Workflow Management
 
-Here we have the start of a system to automatically generated workflows (currently only for transforms). 
+Here we have the start of a system to automatically generated github workflows (currently only for transforms). 
 In general, the design is to use templates and `make` to generate/update the workflows.
 
 ```
@@ -21,7 +21,6 @@ The Makefile is used to (re)generate all workflows a necessary.  By design, work
 
 When a new transform is added to the repository, 
 
-1. add it to the corresponding macro in the Makefile (in alphabetical order please). 
-1. Run `make` to create the new test .yml for the transform
+1. Run `make` in this directory to create the new test .yml for all transforms found in transforms/{universal,code,language} directories 
 1. commit and push the change to your branch with the new transform.
 

.github/workflows/test-code-code2parquet.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/code/code2parquet
 

.github/workflows/test-code-code_quality.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/code/code_quality
 

.github/workflows/test-code-header_cleanser.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/code/header_cleanser
 

.github/workflows/test-code-malware.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/code/malware
 

.github/workflows/test-code-proglang_select.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/code/proglang_select
 

.github/workflows/test-code-repo_level_ordering.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/code/repo_level_ordering
 

.github/workflows/test-language-doc_chunk.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/language/doc_chunk
 

.github/workflows/test-language-doc_quality.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/language/doc_quality
 

.github/workflows/test-language-lang_id.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/language/lang_id
 

.github/workflows/test-language-pdf2parquet.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/language/pdf2parquet
 

.github/workflows/test-language-pii_redactor.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/language/pii_redactor
 

.github/workflows/test-language-text_encoder.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/language/text_encoder
 

.github/workflows/test-transform.template

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - @TARGET_TRANSFORM_DIR@
 

.github/workflows/test-universal-doc_id.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/doc_id
 

.github/workflows/test-universal-ededup.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/ededup
 

.github/workflows/test-universal-fdedup.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/fdedup
 

.github/workflows/test-universal-filter.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/filter
 

.github/workflows/test-universal-html2parquet.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/html2parquet
 

.github/workflows/test-universal-noop.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/noop
 

.github/workflows/test-universal-profiler.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/profiler
 

.github/workflows/test-universal-resize.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/resize
 

.github/workflows/test-universal-tokenization.yml

@@ -1,5 +1,5 @@
 #
-# DO NOT EDIT THIS: it is generated from test-transform.template,  Edit there and run make to change these files
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
 #
 name: Test - transforms/universal/tokenization