add: partial copy method

TOsmanov · TOsmanov · commit 1aab2db07117 · 2025-03-14T13:45:27.000+03:00
diff --git a/foliant/__init__.py b/foliant/__init__.py
@@ -1 +1 @@
-__version__ = '1.0.13'
+__version__ = '1.0.14'
diff --git a/foliant/backends/base.py b/foliant/backends/base.py
@@ -1,10 +1,13 @@
+import re
+import os
 from importlib import import_module
-from shutil import copytree
+from shutil import copytree, copy
+from pathlib import Path
 from datetime import date
 from logging import Logger
-
+from glob import glob
 from foliant.utils import spinner
-
+from typing import Union, List
 
 class BaseBackend():
     '''Base backend. All backends must inherit from this one.'''
@@ -82,6 +85,97 @@ def apply_preprocessor(self, preprocessor: str or dict):
                     f'Failed to apply preprocessor {preprocessor_name}: {exception}'
                 ) from exception
 
+    @staticmethod
+    def partial_copy(
+        source: Union[str, Path, List[Union[str, Path]]],
+        destination: Union[str, Path],
+        root: Union[str, Path]
+    ) -> None:
+        """
+        Copies files, a list of files, or files matching a glob pattern to the specified folder.
+        Creates all necessary directories if they don't exist.
+
+        :param source: A file path, a list of file paths, or a glob pattern (as a string or Path object).
+        :param destination: Target folder (as a string or Path object).
+        :param root: Base folder to calculate relative paths (optional). If not provided, the parent directory of the source is used.
+        """
+        # Convert destination to a Path object
+        destination_path = Path(destination)
+
+        def extract_first_header(file_path):
+            """Extracts the first first-level header from the Markdown file."""
+            with open(file_path, 'r', encoding='utf-8') as file:
+                for line in file:
+                    match = re.match(r'^#\s+(.*)', line)
+                    if match:
+                        return match.group(0)  # Returns the header
+            return None  # If the header is not found
+
+        def copy_files_without_content(src_dir, dst_dir):
+            """Copies files, leaving only the first-level header."""
+            if not os.path.exists(dst_dir):
+                os.makedirs(dst_dir)
+
+            for file_root, _, files in os.walk(src_dir):
+                for file_name in files:
+                    src_file_path = os.path.join(file_root, file_name)
+                    dirs = os.path.relpath(file_root, src_dir)
+                    dst_file_path = Path(os.path.join(dst_dir, dirs, file_name))
+                    dst_file_path.parent.mkdir(parents=True, exist_ok=True)
+                    if file_name.endswith('.md'):
+                        header = extract_first_header(src_file_path)
+                        if header:
+                            with open(dst_file_path, 'w', encoding='utf-8') as dst_file:
+                                dst_file.write(header + '\n')
+                    else:
+                        copy(src_file_path, dst_file_path)
+
+        copy_files_without_content(root, destination)
+        # Handle case where source is a list of files
+        if isinstance(source, str) and ',' in source:
+            print( source)
+            source = source.split(',')
+        if isinstance(source, list):
+            files_to_copy = []
+            for item in source:
+                item_path = Path(item)
+                if not item_path.exists():
+                    raise FileNotFoundError(f"Source '{item}' not found.")
+                files_to_copy.append(item_path)
+        else:
+            # Convert source to a Path object if it's a string
+            if isinstance(source, str):
+                source_path = Path(source)
+            else:
+                source_path = source
+
+            # Check if the source is a glob pattern
+            if isinstance(source, str) and ('*' in source or '?' in source or '[' in source):
+                # Use glob to find files matching the pattern
+                files_to_copy = [Path(file) for file in glob(source, recursive=True)]
+            else:
+                # Check if the source file or directory exists
+                if not source_path.exists():
+                    raise FileNotFoundError(f"Source '{source_path}' not found.")
+                files_to_copy = [source_path]
+
+        # Determine the root directory for calculating relative paths
+        root = Path(root)
+
+        # Copy each file
+        for file_path in files_to_copy:
+            # Calculate the relative path
+            relative_path = file_path.relative_to(root)
+
+            # Full path to the destination file
+            destination_file_path = destination_path / relative_path
+
+            # Create directories if they don't exist
+            destination_file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Copy the file
+            copy(file_path, destination_file_path)
+
     def preprocess_and_make(self, target: str) -> str:
         '''Apply preprocessors required by the selected backend and defined in the config file,
         then run the ``make`` method.
@@ -93,7 +187,10 @@ def preprocess_and_make(self, target: str) -> str:
 
         src_path = self.project_path / self.config['src_dir']
 
-        copytree(src_path, self.working_dir)
+        if self.context['only_partial']:
+            self.partial_copy(self.context['only_partial'], self.working_dir, src_path)
+        else:
+            copytree(src_path, self.working_dir)
 
         common_preprocessors = (
             *self.required_preprocessors_before,
diff --git a/foliant/backends/pre.py b/foliant/backends/pre.py
@@ -23,6 +23,9 @@ def __init__(self, *args, **kwargs):
 
     def make(self, target: str) -> str:
         rmtree(self._preprocessed_dir_name, ignore_errors=True)
-        copytree(self.working_dir, self._preprocessed_dir_name)
+        if self.context['only_partial']:
+            self.partial_copy(self.working_dir / self.context['only_partial'], self._preprocessed_dir_name)
+        else:
+            copytree(self.working_dir, self._preprocessed_dir_name)
 
         return self._preprocessed_dir_name
diff --git a/foliant/cli/make.py b/foliant/cli/make.py
@@ -140,6 +140,7 @@ def get_config(
             'logs_dir': 'Path to the directory to store logs, defaults to project path.',
             'quiet': 'Hide all output accept for the result. Useful for piping.',
             'keep_tmp': 'Keep the tmp directory after the build.',
+            'only_partial': 'using only a partial list of files',
             'debug': 'Log all events during build. If not set, only warnings and errors are logged.'
         }
     )
@@ -152,6 +153,7 @@ def make(
             logs_dir='',
             quiet=False,
             keep_tmp=False,
+            only_partial='',
             debug=False
         ):
         '''Make TARGET with BACKEND.'''
@@ -189,7 +191,9 @@ def make(
             'project_path': project_path,
             'config': config,
             'target': target,
-            'backend': backend
+            'backend': backend,
+            'keep_tmp': keep_tmp,
+            'only_partial': only_partial
         }
 
         backend_module = import_module(f'foliant.backends.{backend}')
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "foliant"
-version = "1.0.13"
+version = "1.0.14"
 description = "Modular, Markdown-based documentation generator that makes pdf, docx, html, and more."
 license = "MIT"
 authors = ["Konstantin Molchanov <moigagoo@live.com>"]
diff --git a/test.sh b/test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# before testing make sure that you have installed the fresh version of preprocessor:
+poetry install --no-interaction
+
+# run tests
+poetry run pytest --cov=foliant -v
diff --git a/test_in_docker.sh b/test_in_docker.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Write Dockerfile
+echo "FROM python:3.9.21-alpine3.20" > Dockerfile
+echo "RUN apk add --no-cache --upgrade bash  && \
+pip install poetry==1 && \
+pip install --no-build-isolation pyyaml==5.4.1" >> Dockerfile
+
+# Run tests in docker
+docker build . -t test-foliant:latest
+
+docker run --rm -it -v "./:/app/" -w /app/ test-foliant:latest "./test.sh"
+
+# Remove Dockerfile
+rm Dockerfile
diff --git a/tests/test_backends.py b/tests/test_backends.py
@@ -0,0 +1,158 @@
+from unittest import TestCase
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from foliant.backends.base import BaseBackend
+
+class TestBackendCopyFiles(TestCase):
+    def setUp(self):
+        # Create a temporary directory for testing
+        self.test_dir = TemporaryDirectory()
+        self.source_dir = Path(self.test_dir.name) / "source"
+        self.destination_dir = Path(self.test_dir.name) / "destination"
+        self.source_dir.mkdir()
+        self.destination_dir.mkdir()
+
+        # Create some test files
+        (self.source_dir / "file1.txt").write_text("Hello, file1!")
+        (self.source_dir / "file2.txt").write_text("Hello, file2!")
+        (self.source_dir / "subfolder").mkdir()
+        (self.source_dir / "subfolder" / "file3.txt").write_text("Hello, file3!")
+        (self.source_dir / "file2.md").write_text("# Header\nSome content")
+        (self.source_dir / "subfolder" / "file3.md").write_text("# Another Header\nMore content")
+
+    def tearDtown(self):
+        # Clean up the temporary directory
+        self.test_dir.cleanup()
+
+    def test_copy_single_file(self):
+        # Test copying a single file
+        source_file = self.source_dir / "file1.txt"
+        BaseBackend.partial_copy(source_file, self.destination_dir, self.source_dir)
+
+        # Check if the file was copied
+        self.assertTrue((self.destination_dir / "file1.txt").exists())
+        self.assertEqual((self.destination_dir / "file1.txt").read_text(), "Hello, file1!")
+
+    def test_copy_list_of_files(self):
+        # Test copying a list of files
+        source_files = [
+            self.source_dir / "file1.txt",
+            self.source_dir / "file2.txt"
+        ]
+        BaseBackend.partial_copy(source_files, self.destination_dir, self.source_dir)
+
+        # Check if the files were copied
+        self.assertTrue((self.destination_dir / "file1.txt").exists())
+        self.assertTrue((self.destination_dir / "file2.txt").exists())
+
+
+    def test_copy_list_of_files_two(self):
+        # Test copying a list of files
+        source_files = [
+            str(self.source_dir / "file1.txt"),
+            str(self.source_dir / "file2.md")
+        ]
+        BaseBackend.partial_copy(source_files, self.destination_dir, self.source_dir)
+
+        # Check if the files were copied
+        self.assertTrue((self.destination_dir / "file1.txt").exists())
+        self.assertTrue((self.destination_dir / "file2.md").exists())
+
+    def test_copy_glob_pattern(self):
+        # Test copying files matching a glob pattern
+        glob_pattern = str(self.source_dir / "*.txt")
+        BaseBackend.partial_copy(glob_pattern, self.destination_dir, self.source_dir)
+
+        # Check if the files were copied
+        self.assertTrue((self.destination_dir / "file1.txt").exists())
+        self.assertTrue((self.destination_dir / "file2.txt").exists())
+        self.assertFalse((self.destination_dir / "file3.txt").exists())  # file3 is in a subfolder
+
+    def test_copy_glob_pattern_md(self):
+        # Test copying files matching a glob pattern
+        glob_pattern = str(self.source_dir / '*2.md')
+        BaseBackend.partial_copy(glob_pattern, self.destination_dir, self.source_dir)
+
+        # Check if the files were copied
+        self.assertTrue((self.destination_dir / "file2.md").exists())
+        self.assertEqual((self.destination_dir / "file2.md").read_text(), "# Header\nSome content")
+        self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists())
+        self.assertEqual((self.destination_dir / "subfolder" / "file3.md").read_text(), "# Another Header\n") # Only '*2.md' files should be copied with content
+
+    def test_copy_glob_pattern_recursive(self):
+        # Test copying files matching a recursive glob pattern
+        glob_pattern = str(self.source_dir / "**" / "*.txt")
+        BaseBackend.partial_copy(glob_pattern, self.destination_dir, self.source_dir)
+
+        # Check if the files were copied, including the one in the subfolder
+        self.assertTrue((self.destination_dir / "file1.txt").exists())
+        self.assertTrue((self.destination_dir / "file2.txt").exists())
+        self.assertTrue((self.destination_dir / "subfolder" / "file3.txt").exists())
+
+    def test_copy_directory_structure(self):
+        # Test copying a file while preserving directory structure
+        source_file = self.source_dir / "subfolder" / "file3.txt"
+        BaseBackend.partial_copy(source_file, self.destination_dir, self.source_dir)
+
+        # Check if the file was copied with the directory structure
+        self.assertTrue((self.destination_dir / "subfolder" / "file3.txt").exists())
+
+    def test_copy_nonexistent_file(self):
+        # Test copying a nonexistent file (should raise FileNotFoundError)
+        source_file = self.source_dir / "nonexistent.txt"
+        with self.assertRaises(FileNotFoundError):
+            BaseBackend.partial_copy(source_file, self.destination_dir, self.source_dir)
+
+    def test_copy_nonexistent_glob(self):
+        # Test copying with a glob pattern that matches no files
+        glob_pattern = str(self.source_dir / "*_suffix.md")
+        BaseBackend.partial_copy(glob_pattern, self.destination_dir, self.source_dir)
+
+        # Check that no files were copied
+        self.assertTrue((self.destination_dir / "file2.md").exists())
+        self.assertEqual((self.destination_dir / "file2.md").read_text(), "# Header\n")
+        self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists())
+        self.assertEqual((self.destination_dir / "subfolder" / "file3.md").read_text(), "# Another Header\n")
+
+    def test_copy_to_nonexistent_destination(self):
+        # Test copying to a nonexistent destination (should create the destination folder)
+        new_destination = self.destination_dir / "new_folder"
+        BaseBackend.partial_copy(self.source_dir / "file1.txt", new_destination, self.source_dir)
+
+        # Check if the file was copied and the destination folder was created
+        self.assertTrue(new_destination.exists())
+        self.assertTrue((new_destination / "file1.txt").exists())
+
+    def test_copy_path_object(self):
+        # Test copying using Path objects
+        source_file = self.source_dir / "file1.txt"
+        destination = self.destination_dir / "file1.txt"
+        BaseBackend.partial_copy(source_file, destination, self.source_dir)
+
+        # Check if the file was copied
+        self.assertTrue(destination.exists())
+
+    def test_copy_text_file(self):
+        # Test copying a text file
+        BaseBackend.partial_copy(str(self.source_dir / "file1.txt"), self.destination_dir, self.source_dir)
+
+        # Check if the file was copied
+        self.assertTrue((self.destination_dir / "file1.txt").exists())
+        self.assertEqual((self.destination_dir / "file1.txt").read_text(), "Hello, file1!")
+
+    def test_copy_markdown_file(self):
+        # Test copying a Markdown file
+        BaseBackend.partial_copy(str(self.source_dir / "file2.md"), self.destination_dir, self.source_dir)
+
+        # Check if only the header was copied
+        self.assertTrue((self.destination_dir / "file2.md").exists())
+        self.assertEqual((self.destination_dir / "file2.md").read_text(), "# Header\nSome content")
+
+    def test_copy_directory_structure_with_header(self):
+        # Test copying a file while preserving directory structure
+        BaseBackend.partial_copy(str(self.source_dir / "subfolder" / "file3.md"), self.destination_dir, self.source_dir)
+
+        # Check if the file was copied with the directory structure
+        self.assertTrue((self.destination_dir / "subfolder" / "file3.md").exists())
+        self.assertEqual((self.destination_dir / "subfolder" / "file3.md").read_text(), "# Another Header\nMore content")

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = '1.0.13'`
	`1`	`+__version__ = '1.0.14'`