From 09a93a2e75969b7ebf977f3ed51bc1f0ecd681a5 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 12:55:29 +0000 Subject: [PATCH 01/10] Disk checkpoint storing all time step data --- checkpoint_schedules/basic_schedules.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/checkpoint_schedules/basic_schedules.py b/checkpoint_schedules/basic_schedules.py index 67d74c8..390952f 100644 --- a/checkpoint_schedules/basic_schedules.py +++ b/checkpoint_schedules/basic_schedules.py @@ -89,7 +89,7 @@ def uses_storage_type(self, storage_type): class SingleDiskStorageSchedule(CheckpointSchedule): - """A checkpointing schedule where all adjoint dependencies are stored on + """A checkpointing schedule where all time step data are stored on disk. Notes @@ -101,6 +101,10 @@ class SingleDiskStorageSchedule(CheckpointSchedule): move_data : bool Indicate whether the execution should move the data from `StorageType.DISK` to `StorageType.WORK`, rather than copy the data. + store_only_adj_deps : bool + Indicate whether only adjoint dependencies should be stored on disk. + This is useful when the forward solver identifies the data needed + for the adjoint-based gradient calculations. Notes ----- @@ -108,10 +112,11 @@ class SingleDiskStorageSchedule(CheckpointSchedule): one adjoint calculation permitted if `move_data` is `True`. """ - def __init__(self, move_data=False): + def __init__(self, move_data=False, store_only_adj_deps=False): super().__init__() self._move_data = move_data self._storage = StorageType.DISK + self._store_only_adj_deps = store_only_adj_deps def _iterator(self): """Schedule iterator. @@ -125,7 +130,10 @@ def _iterator(self): n0 = self._n n1 = n0 + 1 self._n = n1 - yield Forward(n0, n1, False, True, StorageType.DISK) + if self._store_only_adj_deps: + yield Forward(n0, n1, False, True, StorageType.DISK) + else: + yield Forward(n0, n1, True, True, StorageType.DISK) yield EndForward() From 4f41d9a2edbf01c9601bd106fb07f5e3cfaef1c0 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 16:33:55 +0000 Subject: [PATCH 02/10] Drop python 3.8 --- .github/workflows/build.yml | 2 +- .github/workflows/lint.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bcf9989..1b9b3a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3d59d66..b2c266d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Setup flake8 annotations uses: rbialon/flake8-annotations@v1 From 073880e6d3c33426567c421114ba605ac22b501b Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 16:34:25 +0000 Subject: [PATCH 03/10] Drop python 3.8 in toml file --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9e8f65f..e705df0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ authors = [ description = "Schedules for incremental checkpointing of adjoint simulations." readme = "README.md" license = {text = "LGPL-3.0"} -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", ] From 871ca29e6ba7835e97c7835fe4c7d3ab74084e85 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 16:37:03 +0000 Subject: [PATCH 04/10] New python versions --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1b9b3a9..e338a6e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 From d4a549bbd034cafbbbf0cdb8314527aad2849a1b Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 16:41:10 +0000 Subject: [PATCH 05/10] Update artifact versions --- .github/workflows/docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1c0132d..78f51bb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -59,7 +59,7 @@ jobs: cd docs cp build/latex/checkpoint_schedules.pdf build/html/_static/manual.pdf - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v3 with: name: github-pages path: /__w/checkpoint_schedules/checkpoint_schedules/docs/build/html @@ -78,4 +78,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@v4 From 29f60cfa34f8fff17c6679f7f546433354043c9d Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 16:59:44 +0000 Subject: [PATCH 06/10] Test for store_only_adj_deps True --- tests/test_validity.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_validity.py b/tests/test_validity.py index b8639b4..9007d91 100644 --- a/tests/test_validity.py +++ b/tests/test_validity.py @@ -85,14 +85,16 @@ def mixed(n, s): {StorageType.RAM: 0, StorageType.DISK: s}, 1) -def single_disk_copy(n, s): - cp_schedule = SingleDiskStorageSchedule(move_data=False) +def single_disk_copy(n, s, store_only_adj_deps=True): + cp_schedule = SingleDiskStorageSchedule( + store_only_adj_deps=store_only_adj_deps, move_data=False) return (cp_schedule, {StorageType.RAM: 0, StorageType.DISK: n}, 1) -def single_disk_move(n, s): - cp_schedule = SingleDiskStorageSchedule(move_data=True) +def single_disk_move(n, s, store_only_adj_deps=True): + cp_schedule = SingleDiskStorageSchedule( + store_only_adj_deps=store_only_adj_deps, move_data=True) return (cp_schedule, {StorageType.RAM: 0, StorageType.DISK: n}, 1) From 3ccc0753be8d86f860af132cb525236ccf0f6074 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 17:06:20 +0000 Subject: [PATCH 07/10] store_only_adj_deps=True as default --- checkpoint_schedules/basic_schedules.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/checkpoint_schedules/basic_schedules.py b/checkpoint_schedules/basic_schedules.py index 390952f..ef85a74 100644 --- a/checkpoint_schedules/basic_schedules.py +++ b/checkpoint_schedules/basic_schedules.py @@ -103,8 +103,6 @@ class SingleDiskStorageSchedule(CheckpointSchedule): `StorageType.DISK` to `StorageType.WORK`, rather than copy the data. store_only_adj_deps : bool Indicate whether only adjoint dependencies should be stored on disk. - This is useful when the forward solver identifies the data needed - for the adjoint-based gradient calculations. Notes ----- @@ -112,7 +110,7 @@ class SingleDiskStorageSchedule(CheckpointSchedule): one adjoint calculation permitted if `move_data` is `True`. """ - def __init__(self, move_data=False, store_only_adj_deps=False): + def __init__(self, move_data=False, store_only_adj_deps=True): super().__init__() self._move_data = move_data self._storage = StorageType.DISK From 849fe4fe85571bd10812443dd0f1b5cf74e092f2 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 17:07:20 +0000 Subject: [PATCH 08/10] wip --- tests/test_validity.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_validity.py b/tests/test_validity.py index 9007d91..8a0ced9 100644 --- a/tests/test_validity.py +++ b/tests/test_validity.py @@ -85,16 +85,14 @@ def mixed(n, s): {StorageType.RAM: 0, StorageType.DISK: s}, 1) -def single_disk_copy(n, s, store_only_adj_deps=True): - cp_schedule = SingleDiskStorageSchedule( - store_only_adj_deps=store_only_adj_deps, move_data=False) +def single_disk_copy(n, s): + cp_schedule = SingleDiskStorageSchedule(store_only_adj_deps=True, move_data=False) return (cp_schedule, {StorageType.RAM: 0, StorageType.DISK: n}, 1) -def single_disk_move(n, s, store_only_adj_deps=True): - cp_schedule = SingleDiskStorageSchedule( - store_only_adj_deps=store_only_adj_deps, move_data=True) +def single_disk_move(n, s): + cp_schedule = SingleDiskStorageSchedule(move_data=True) return (cp_schedule, {StorageType.RAM: 0, StorageType.DISK: n}, 1) From b83014c9aeca9a2862212d00383bb7ab407cdf95 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 17:07:28 +0000 Subject: [PATCH 09/10] wip --- tests/test_validity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_validity.py b/tests/test_validity.py index 8a0ced9..b8639b4 100644 --- a/tests/test_validity.py +++ b/tests/test_validity.py @@ -86,7 +86,7 @@ def mixed(n, s): def single_disk_copy(n, s): - cp_schedule = SingleDiskStorageSchedule(store_only_adj_deps=True, move_data=False) + cp_schedule = SingleDiskStorageSchedule(move_data=False) return (cp_schedule, {StorageType.RAM: 0, StorageType.DISK: n}, 1) From 760dbd8f223ed2096c38ec383e08f9ebc81d6aa9 Mon Sep 17 00:00:00 2001 From: Iglesia Dolci Date: Thu, 20 Mar 2025 17:23:25 +0000 Subject: [PATCH 10/10] Right docs --- checkpoint_schedules/basic_schedules.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/checkpoint_schedules/basic_schedules.py b/checkpoint_schedules/basic_schedules.py index ef85a74..75574c3 100644 --- a/checkpoint_schedules/basic_schedules.py +++ b/checkpoint_schedules/basic_schedules.py @@ -89,11 +89,9 @@ def uses_storage_type(self, storage_type): class SingleDiskStorageSchedule(CheckpointSchedule): - """A checkpointing schedule where all time step data are stored on + """A checkpointing schedule where all adjoint dependencies are stored on disk. - Notes - ----- Online, unlimited adjoint calculations permitted. Parameters @@ -108,6 +106,9 @@ class SingleDiskStorageSchedule(CheckpointSchedule): ----- Online, unlimited adjoint calculations permitted if `move_data` is `False`, one adjoint calculation permitted if `move_data` is `True`. + + ``store_only_adj_deps`` set as False will store both all forward data on + disk. """ def __init__(self, move_data=False, store_only_adj_deps=True):