Generalize SingleYearDataset and MultipleYearDataset classes

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install Python dependencies
         run: |
-          uv pip install -e ".[dev]" --system
+          uv pip install -e ".[dev,docs]" --system
       - name: Run tests with coverage
         run: make test
       - name: Upload coverage to Codecov
@@ -31,28 +31,20 @@ jobs:
           file: ./coverage.xml
           fail_ci_if_error: false
           verbose: true
-  publish-to-pypi:
-    name: Publish to PyPI
-    needs: Test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # Fetch all history for all tags and branches
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install package
-        run: make install
-      - name: Build package
-        run: python -m build
-      - name: Publish a git tag
-        run: ".github/publish-git-tag.sh || true"
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Test documentation builds
+        run: make documentation
+      - name: Check documentation build
+        run: |
+          for notebook in $(find docs/_build/jupyter_execute -name "*.ipynb"); do
+            if grep -q '"output_type": "error"' "$notebook"; then
+              echo "Error found in $notebook"
+              cat "$notebook"
+              exit 1
+            fi
+          done
+      - name: Deploy documentation
+        uses: JamesIves/github-pages-deploy-action@releases/v3
         with:
-          user: __token__
-          password: ${{ secrets.PYPI }}
-          skip-existing: true
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: gh-pages  # The branch the action should deploy to.
+          FOLDER: docs/_build/html  # The folder the action should deploy.

.github/workflows/pr.yaml

@@ -61,6 +61,17 @@ jobs:
           python-version: "3.11"
       - name: Install dependencies
         run: |
-          uv pip install -e ".[dev]" --system
+          uv pip install -e ".[dev,docs]" --system
       - name: Build package
         run: make build
+      - name: Test documentation builds
+        run: make documentation
+      - name: Check documentation build
+        run: |
+          for notebook in $(find docs/_build/jupyter_execute -name "*.ipynb"); do
+            if grep -q '"output_type": "error"' "$notebook"; then
+              echo "Error found in $notebook"
+              cat "$notebook"
+              exit 1
+            fi
+          done

.github/workflows/versioning.yaml

@@ -35,4 +35,28 @@ jobs:
               with:
                 add: "."
                 message: Update package version
-
+    publish-to-pypi:
+      name: Publish to PyPI
+      if: (github.event.head_commit.message == 'Update package version')
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v4
+          with:
+            fetch-depth: 0 # Fetch all history for all tags and branches
+        - name: Set up Python
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python-version }}
+        - name: Install package
+          run: make install
+        - name: Build package
+          run: python -m build
+        - name: Publish a git tag
+          run: ".github/publish-git-tag.sh || true"
+        - name: Publish to PyPI
+          uses: pypa/gh-action-pypi-publish@release/v1
+          with:
+            user: __token__
+            password: ${{ secrets.PYPI }}
+            skip-existing: true

.gitignore

@@ -205,3 +205,9 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+
+# Data files
+*.csv
+*.h5
+
+.DS_Store

changelog.yaml

@@ -1,5 +1,5 @@
 - changes:
     added:
     - Initialized project.
-  date: 2025-07-22
+  date: 2025-07-22 00:00:00
   version: 0.1.0

changelog_entry.yaml

@@ -1,6 +1,5 @@
 - bump: minor
   changes:
     changed:
-    - Initialized changelogging.
-    - Added CI workflows and tests.
-    - Set up basic package structure and coverage.
+    - Added Single and Multi Year Dataset classes.
+    - Added data download and upload functionality.

docs/_config.yml

@@ -0,0 +1,26 @@
+title: Policyengine-data documentation
+author: PolicyEngine
+logo: logo.png
+
+execute:
+  execute_notebooks: force
+  timeout: 180
+
+repository:
+  url: https://github.com/policyengine/policyengine-data
+  branch: main
+  path_to_book: docs
+
+sphinx:
+  config:
+    html_js_files:
+      - https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js
+    html_theme: furo
+    pygments_style: default
+    html_css_files:
+      - style.css
+  extra_extensions:
+    - "sphinx.ext.autodoc"
+    - "sphinx.ext.viewcode"
+    - "sphinx.ext.napoleon"
+    - "sphinx.ext.mathjax"

docs/_static/__init__.py

@@ -0,0 +1 @@
+

docs/_toc.yml

@@ -0,0 +1,4 @@
+format: jb-book
+root: intro
+chapters:
+  - file: dataset.ipynb

docs/add_plotly_to_book.py

@@ -0,0 +1,27 @@
+import argparse
+from pathlib import Path
+
+# This command-line tools enables Plotly charts to show in the HTML files for the Jupyter Book documentation.
+
+parser = argparse.ArgumentParser()
+parser.add_argument("book_path", help="Path to the Jupyter Book.")
+
+args = parser.parse_args()
+
+# Find every HTML file in the Jupyter Book. Then, add a script tag to the start of the <head> tag in each file, with the contents:
+# <script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
+
+book_folder = Path(args.book_path)
+
+for html_file in book_folder.glob("**/*.html"):
+    with open(html_file, "r") as f:
+        html = f.read()
+
+    # Add the script tag to the start of the <head> tag.
+    html = html.replace(
+        "<head>",
+        '<head><script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>',
+    )
+
+    with open(html_file, "w") as f:
+        f.write(html)