Feature/save-raw-report-data

.github/workflows/python-app.yml

@@ -9,42 +9,31 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - uses: actions/checkout@v6
+    - name: Install uv
+      uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
       with:
         python-version: "3.10"
+        enable-cache: true
 
-    - name: Install nox
-      run: |
-          python -m pip install nox
-
-    - name: Install poetry
-      run: |
-          python -m pip install poetry
+    - name: Install project
+      run: uv sync --all-extras --dev
 
-    - name: Install dependencies
-      run: |
-        poetry install --with dev
-
-    - name: Make sure the code can run
-      run: |
-        # cd src; poetry run python -m folio_migration_tools -h
+    - name: Lint with Ruff
+      run: uv run ruff check src/
 
-    - name: Run the nox test session
+    - name: Run tests
       env:
         GITHUB_TOKEN: ${{ secrets.TESTS_GITHUB_TOKEN }}
+      run: uv run pytest --cov=./ --cov-report=xml --durations=20 tests/
+
+    - name: Make sure the code can run
       run: |
-        nox -rs tests
-
-    - name: Run the nox lint session
-      run: |
-        nox -rs lint
+        uv run folio-migration-tools -h
 
     - name: Run the nox safety session
       run: |
-        nox -rs safety
-
+        uvx nox -rs safety
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3.1.4        

.github/workflows/python-publish.yml

@@ -12,23 +12,30 @@ jobs:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+    - name: Install uv
+      uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
       with:
         python-version: ${{ matrix.python-version }}
-        cache: pip
-        cache-dependency-path: '**/pyproject.toml'
-    - name: Install poetry
-      run: |
-        python -m pip install poetry
-    - name: Install poetry
-      run: |
-        poetry install --with dev        
+        enable-cache: true
+
+    - name: Install project
+      run: uv sync --all-extras --dev
+
+    - name: Lint with Ruff
+      run: uv run ruff check src/
+
     - name: Run tests
       env:
         GITHUB_TOKEN: ${{ secrets.TESTS_GITHUB_TOKEN }}
+      run: uv run pytest --cov=./ --cov-report=xml --durations=20 tests/
+
+    - name: Make sure the code can run
       run: |
-        poetry run pytest
+        uv run folio-migration-tools -h
+
+    - name: Run the nox safety session
+      run: |
+        uvx nox -rs safety
   deploy:
     runs-on: ubuntu-latest
     needs: [test]
@@ -37,17 +44,16 @@ jobs:
       id-token: write
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v5
+    - name: Install uv
+      uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b
       with:
-        python-version: '3.13'
-        cache: pip
-        cache-dependency-path: '**/pyproject.toml'
-    - name: Install dependencies
-      run: |
-        python -m pip install poetry
-    - name: Build
+        python-version: "3.13"
+        enable-cache: true
+    - name: Install project
+      run: uv sync --all-extras --dev
+
+    - name: Build wheel
       run: |
-        poetry build
+        uv build
     - name: Publish
-      uses: pypa/gh-action-pypi-publish@release/v1
+      run: uv publish

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "folio_migration_tools"
-version = "1.10.0b6"
+version = "1.10.0b7"
 description =  "A tool allowing you to migrate data from legacy ILS:s (Library systems) into FOLIO LSP"
 authors = [
 	{name = "Theodor Tolstoy", email = "github.teddes@tolstoy.se"},
@@ -48,6 +48,7 @@ dev = [
 	"pytest-asyncio>=0.23.0,<0.24.0",
 	"lxml>=6.0.2",
 	"pyrefly>=0.43.1",
+	"ruff>=0.14.13",
 ]
 docs = [
 	"m2r>=0.2.1,<0.3.0",

scripts/extract_translations.py

@@ -57,7 +57,8 @@
             for subkey in translations[key]:
                 if missing_format_re.search(translations[key][subkey]):
                     print(
-                        f"Key '{key}' plural '{subkey}' may not format correctly: format must have %"
+                        f"Key '{key}' plural '{subkey}' may not format "
+                        f"correctly: format must have %"
                     )
     # Write
     with open(en_filename, "w") as f:

scripts/update_language.py

@@ -39,7 +39,8 @@
     for key in target_translations:
         if key not in source_translations:
             print(
-                f"Key '{key}' in target not in source. Check if it was renamed, or if it is still needed."
+                f"Key '{key}' in target not in source. "
+                f"Check if it was renamed, or if it is still needed."
             )
     # Update target translations
     for key in source_translations:
@@ -62,7 +63,8 @@
             for subkey in target_translations[key]:
                 if missing_format_re.search(target_translations[key][subkey]):
                     print(
-                        f"Key '{key}' plural '{subkey}' may not format correctly: format must have %"
+                        f"Key '{key}' plural '{subkey}' may not format "
+                        f"correctly: format must have %"
                     )
     # Write
     with open(target_filename, "w") as f:

src/folio_migration_tools/circulation_helper.py

@@ -11,6 +11,7 @@
 from folioclient import FolioClient, FolioClientError, FolioConnectionError, FolioValidationError
 
 from folio_migration_tools.helper import Helper
+from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.migration_report import MigrationReport
 from folio_migration_tools.transaction_migration.legacy_loan import LegacyLoan
 from folio_migration_tools.transaction_migration.legacy_request import LegacyRequest
@@ -37,7 +38,7 @@ def __init__(
     def get_user_by_barcode(self, user_barcode):
         if user_barcode in self.missing_patron_barcodes:
             self.migration_report.add_general_statistics(
-                i18n.t("Users already detected as missing")
+                i18n_t("Users already detected as missing")
             )
             logging.info("User is already detected as missing")
             return {}
@@ -55,7 +56,7 @@ def get_user_by_barcode(self, user_barcode):
     def get_item_by_barcode(self, item_barcode):
         if item_barcode in self.missing_item_barcodes:
             self.migration_report.add_general_statistics(
-                i18n.t("Items already detected as missing")
+                i18n_t("Items already detected as missing")
             )
             logging.info("Item is already detected as missing")
             return {}
@@ -140,7 +141,7 @@ def check_out_by_barcode(self, legacy_loan: LegacyLoan) -> TransactionResult:
         path = "/circulation/check-out-by-barcode"
         try:
             if legacy_loan.patron_barcode in self.missing_patron_barcodes:
-                error_message = i18n.t("Patron barcode already detected as missing")
+                error_message = i18n_t("Patron barcode already detected as missing")
                 logging.error(
                     f"{error_message} Patron barcode: {legacy_loan.patron_barcode} "
                     f"Item Barcode:{legacy_loan.item_barcode}"
@@ -189,7 +190,7 @@ def check_out_by_barcode(self, legacy_loan: LegacyLoan) -> TransactionResult:
             elif "find user with matching barcode" in error_message_from_folio:
                 self.missing_patron_barcodes.add(legacy_loan.patron_barcode)
                 error_message = f"No patron with barcode {legacy_loan.patron_barcode} in FOLIO"
-                stat_message = i18n.t("Patron barcode not in FOLIO")
+                stat_message = i18n_t("Patron barcode not in FOLIO")
                 return TransactionResult(
                     False,
                     False,
@@ -248,7 +249,7 @@ def check_out_by_barcode(self, legacy_loan: LegacyLoan) -> TransactionResult:
                 False,
                 None,
                 "Connection error",
-                i18n.t("Connection error during checkout"),
+                i18n_t("Connection error during checkout"),
             )
 
     @staticmethod

src/folio_migration_tools/folder_structure.py

@@ -22,6 +22,8 @@ def __init__(
         self.add_time_stamp_to_file_names = add_time_stamp_to_file_names
         self.iteration_identifier = iteration_identifier
         self.base_folder = Path(base_path)
+        # Ensure the base folder exists and is a directory. This differs from other folders, which
+        # are created if missing.
         if not self.base_folder.is_dir():
             logging.critical("Base Folder Path is not a folder. Exiting.")
             sys.exit(1)
@@ -43,6 +45,10 @@ def __init__(
         self.reports_folder = self.iteration_folder / "reports"
         self.verify_folder(self.reports_folder)
 
+        # Raw migration reports directory
+        self.raw_reports_folder = self.reports_folder / ".raw"
+        self.verify_folder(self.raw_reports_folder)
+
     def log_folder_structure(self):
         logging.info("Mapping files folder is %s", self.mapping_files_folder)
         logging.info("Git ignore is set up correctly")
@@ -98,6 +104,10 @@ def setup_migration_file_structure(self, source_file_type: str = ""):
 
         self.migration_reports_file = self.reports_folder / f"report{self.file_template}.md"
 
+        self.migration_reports_raw_file = (
+            self.raw_reports_folder / f"raw_report{self.file_template}.json"
+        )
+
         self.srs_records_path = (
             self.results_folder / f"folio_srs_{object_type_string}{self.file_template}.json"
         )
@@ -128,10 +138,13 @@ def setup_migration_file_structure(self, source_file_type: str = ""):
         self.item_statuses_map_path = self.mapping_files_folder / "item_statuses.tsv"
 
     def verify_folder(self, folder_path: Path):
-        if not folder_path.is_dir():
-            logging.critical("There is no folder located at %s. Exiting.", folder_path)
-            logging.critical("Create a folder by calling\n\tmkdir %s", folder_path)
+        if folder_path.exists() and not folder_path.is_dir():
+            logging.critical("Path exists but is not a directory: %s", folder_path)
             sys.exit(1)
+
+        if not folder_path.exists():
+            logging.info("Creating missing folder %s", folder_path)
+            folder_path.mkdir(parents=True, exist_ok=True)
         else:
             logging.info("Located %s", folder_path)
 

src/folio_migration_tools/helper.py

@@ -1,6 +1,7 @@
 import json
 import logging
-import i18n
+
+from folio_migration_tools.i18n_cache import i18n_t
 
 
 class Helper:
@@ -9,15 +10,15 @@ def print_mapping_report(
         report_file, total_records: int, mapped_folio_fields, mapped_legacy_fields
     ):
         details_start = (
-            "<details><summary>" + i18n.t("Click to expand field report") + "</summary>\n\n"
+            "<details><summary>" + i18n_t("Click to expand field report") + "</summary>\n\n"
         )
         details_end = "</details>\n"
-        report_file.write("\n## " + i18n.t("Mapped FOLIO fields") + "\n")
+        report_file.write("\n## " + i18n_t("Mapped FOLIO fields") + "\n")
         # report_file.write(f"{blurbs[header]}\n")
 
         d_sorted = {k: mapped_folio_fields[k] for k in sorted(mapped_folio_fields)}
         report_file.write(details_start)
-        columns = [i18n.t("FOLIO Field"), i18n.t("Mapped"), i18n.t("Unmapped")]
+        columns = [i18n_t("FOLIO Field"), i18n_t("Mapped"), i18n_t("Unmapped")]
         report_file.write(" | ".join(columns) + "\n")
         report_file.write("|".join(len(columns) * ["---"]) + "\n")
         for k, v in d_sorted.items():
@@ -32,12 +33,12 @@ def print_mapping_report(
             )
         report_file.write(details_end)
 
-        report_file.write("\n## " + i18n.t("Mapped Legacy fields") + "\n")
+        report_file.write("\n## " + i18n_t("Mapped Legacy fields") + "\n")
         # report_file.write(f"{blurbs[header]}\n")
 
         d_sorted = {k: mapped_legacy_fields[k] for k in sorted(mapped_legacy_fields)}
         report_file.write(details_start)
-        columns = [i18n.t("Legacy Field"), i18n.t("Present"), i18n.t("Mapped"), i18n.t("Unmapped")]
+        columns = [i18n_t("Legacy Field"), i18n_t("Present"), i18n_t("Mapped"), i18n_t("Unmapped")]
         report_file.write("|".join(columns) + "\n")
         report_file.write("|".join(len(columns) * ["---"]) + "\n")
         for k, v in d_sorted.items():

src/folio_migration_tools/holdings_helper.py

@@ -5,6 +5,7 @@
 
 from folio_migration_tools import custom_exceptions
 from folio_migration_tools import helper
+from folio_migration_tools.i18n_cache import i18n_t
 from folio_migration_tools.migration_report import MigrationReport
 
 
@@ -54,7 +55,7 @@ def to_key(
                 values.append(str(uuid4()))
                 migration_report.add(
                     "HoldingsMerging",
-                    i18n.t("Holding prevented from merging by holdingsTypeId"),
+                    i18n_t("Holding prevented from merging by holdingsTypeId"),
                 )
             return "-".join(values)
         except Exception as exception:
@@ -99,12 +100,12 @@ def load_previously_generated_holdings(
                     )
                     migration_report.add(
                         "HoldingsMerging",
-                        i18n.t("Duplicate key based on current merge criteria. Records merged"),
+                        i18n_t("Duplicate key based on current merge criteria. Records merged"),
                     )
                 else:
                     migration_report.add(
                         "HoldingsMerging",
-                        i18n.t("Previously transformed holdings record loaded"),
+                        i18n_t("Previously transformed holdings record loaded"),
                     )
                     prev_holdings[stored_key] = stored_holding
             return prev_holdings