Improved onboard_project.py (#783)

* Improved onboard_project.py

-Added collect_verse_counts to onboard_project
-Replaced command line args with a yaml config

* Updated onboard_project

* Made --config optional and added args

-Made --extract-corpora and --collect-verse-counts args without requiring a config, using default values to run both tasks.

* Fixed some bugs with collect_verse_counts defaults

* Check project files before collect_verse_counts

Author: TaperChipmunk32

silnlp/common/collect_verse_counts.py

@@ -264,10 +264,8 @@ def main() -> None:
     # If the output folder doesn't exist locally, assume it's an experiment folder
     folder = args.folder.replace("\\", "/")
     if Path(folder).exists():
-        exp_name = None
         folder = Path(folder)
     else:
-        exp_name = folder
         folder = get_mt_exp_dir(folder)
 
     # If no files are listed and folder is an experiment, use the files listed in the config file

silnlp/common/extract_corpora.py

@@ -105,6 +105,7 @@ def extract_corpora(
                 extract_lemmas,
                 extract_project_vrefs,
             )
+            LOGGER.info(f"Extracted corpus file: {corpus_filename}")
             # check if the number of lines in the file is correct (the same as vref.txt)
             LOGGER.info(f"# of Verses: {verse_count}")
             if verse_count != expected_verse_count:

silnlp/common/onboard_project.py

@@ -2,7 +2,11 @@
 import logging
 from pathlib import Path
 
+import yaml
+
+from .collect_verse_counts import collect_verse_counts
 from .environment import SIL_NLP_ENV
+from .extract_corpora import extract_corpora
 
 LOGGER = logging.getLogger(__package__ + ".onboard_project")
 
@@ -62,70 +66,114 @@ def main() -> None:
     parser.add_argument(
         "project",
         help="Paratext project name. The project will be stored on the bucket at Paratext/projects/<project>.",
+        type=str,
     )
     parser.add_argument(
         "--copy-from",
         help="Path to a downloaded Paratext project folder. The local project will be copied to the bucket.",
         default=None,
+        type=str,
     )
     parser.add_argument(
-        "--overwrite", help="Overwrite any existing files and folders", default=False, action="store_true"
-    )
-    parser.add_argument(
-        "--extract-corpora",
-        help="Extract text corpora.",
-        default=False,
-        action="store_true",
-    )
-    parser.add_argument(
-        "--include",
-        metavar="books",
-        nargs="+",
-        default=[],
-        help="The books to include; e.g., 'NT', 'OT', 'GEN'. Only used with extract-corpora.",
-    )
-    parser.add_argument(
-        "--exclude",
-        metavar="books",
-        nargs="+",
-        default=[],
-        help="The books to exclude; e.g., 'NT', 'OT', 'GEN'. Only used with extract-corpora.",
+        "--config",
+        help="Path to a configuration file in YAML format. This is used to configure the onboarding process.",
+        default=None,
+        type=str,
     )
     parser.add_argument(
-        "--markers", default=False, action="store_true", help="Include USFM markers. Only used with extract-corpora."
+        "--overwrite", help="Overwrite any existing files and folders", default=False, action="store_true"
     )
+
     parser.add_argument(
-        "--lemmas",
+        "--extract-corpora",
         default=False,
         action="store_true",
-        help="Extract lemmas if available. Only used with extract-corpora.",
+        help="Extract text corpora from the Paratext project.",
     )
+
     parser.add_argument(
-        "--project-vrefs",
+        "--collect-verse-counts",
         default=False,
         action="store_true",
-        help="Extract project verse refs. Only used with extract-corpora.",
+        help="Collect various counts from the extracted Paratext project.",
     )
 
     args = parser.parse_args()
-    project_name = args.project
+    if not args.project:
+        raise ValueError("Project name is required. Please provide a valid Paratext project name using <project>.")
 
-    LOGGER.info(f"Onboarding project: {args.project}")
-    paratext_project_dir: Path = create_paratext_project_folder_if_not_exists(project_name)
+    project_name = args.project
 
     if args.copy_from:
+        LOGGER.info(f"Onboarding project: {args.project}")
+        paratext_project_dir: Path = create_paratext_project_folder_if_not_exists(project_name)
         copy_paratext_project_folder(Path(args.copy_from), paratext_project_dir, overwrite=args.overwrite)
 
-    if args.extract_corpora:
-        from .extract_corpora import extract_corpora
+    if args.config:
+        config_file = Path(args.config)
+        if not config_file.exists():
+            raise FileNotFoundError(f"Config file '{config_file}' does not exist.")
+        with config_file.open("r", encoding="utf-8") as file:
+            config = yaml.safe_load(file)
+    else:
+        raise ValueError("Config file is required. Please provide a valid configuration file using --config.")
 
+    if args.extract_corpora:
+        LOGGER.info(f"Extracting {project_name}.")
         extract_corpora(
             projects={project_name},
-            books_to_include=args.include,
-            books_to_exclude=args.exclude,
-            include_markers=args.markers,
-            extract_lemmas=args.lemmas,
-            extract_project_vrefs=args.project_vrefs,
+            books_to_include=config["extract_corpora"]["include"] if "include" in config["extract_corpora"] else [],
+            books_to_exclude=config["extract_corpora"]["exclude"] if "exclude" in config["extract_corpora"] else [],
+            include_markers=(config["extract_corpora"]["markers"] if "markers" in config["extract_corpora"] else False),
+            extract_lemmas=config["extract_corpora"]["lemmas"] if "lemmas" in config["extract_corpora"] else False,
+            extract_project_vrefs=(
+                config["extract_corpora"]["project-vrefs"] if "project-vrefs" in config["extract_corpora"] else False
+            ),
+        )
+
+    if args.collect_verse_counts:
+        if not args.extract_corpora:
+            LOGGER.warning(
+                "--extract_corpora was not included. Collecting verse counts requires the corpus to be extracted first."
+            )
+
+        LOGGER.info(f"Collecting verse counts from {project_name}.")
+
+        if config["verse_counts"]["output_folder"]:
+            output_folder = Path(config["verse_counts"]["output_folder"])
+            if not output_folder.exists():
+                output_folder.mkdir(parents=True, exist_ok=True)
+        else:
+            output_folder = SIL_NLP_ENV.mt_experiments_dir / "verse_counts" / project_name
+            if not output_folder.exists():
+                output_folder.mkdir(parents=True, exist_ok=True)
+        input_folder = (
+            config["verse_counts"]["input_folder"]
+            if "input_folder" in config["verse_counts"]
+            else SIL_NLP_ENV.mt_scripture_dir
+        )
+        file_patterns = (
+            config["verse_counts"]["files"] if "files" in config["verse_counts"] else f"*{project_name}*.txt"
+        )
+
+        input_folder_path = Path(input_folder)
+        if not input_folder_path.exists():
+            LOGGER.error(f"Input folder '{input_folder_path}' does not exist. Skipping verse counts collection.")
+            return
+
+        matched_files = list(input_folder_path.glob(file_patterns))
+        if not matched_files:
+            LOGGER.error(
+                f"No files matching pattern '{file_patterns}' found in '{input_folder_path}'. Skipping verse counts collection."
+            )
+            return
+
+        collect_verse_counts(
+            input_folder=input_folder_path,
+            output_folder=output_folder,
+            file_patterns=file_patterns,
+            deutero=config["verse_counts"]["deutero"] if "deutero" in config["verse_counts"] else False,
+            recount=config["verse_counts"]["recount"] if "recount" in config["verse_counts"] else False,
         )