Repo setup

.github/CONTRIBUTING.md

@@ -0,0 +1,7 @@
+## Updating the code
+
+Please make sure that introduced changes are consistent with the testing api and add additional tests if relevant.
+
+## Updating the versioning
+
+Please add to `changelog.yaml` and then run `make changelog` before committing the results ONCE in this PR.

.github/changelog_template.md

@@ -0,0 +1,8 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+{{changelog}}

.github/get-changelog-diff.sh

@@ -0,0 +1,2 @@
+last_tagged_commit=`git describe --tags --abbrev=0 --first-parent`
+git --no-pager diff $last_tagged_commit -- CHANGELOG.md

.github/has-functional-changes.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile docs/* .gitignore LICENSE* .github/* data/*"
+
+last_tagged_commit=`git describe --tags --abbrev=0 --first-parent`  # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit
+
+if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'`  # Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published.
+then
+  echo "No functional changes detected."
+  exit 1
+else echo "The functional files above were changed."
+fi

.github/is-version-number-acceptable.sh

@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+
+if [[ ${GITHUB_REF#refs/heads/} == master ]]
+then
+    echo "No need for a version check on master."
+    exit 0
+fi
+
+if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh
+then
+    echo "No need for a version update."
+    exit 0
+fi
+
+current_version=`python .github/fetch_version.py`
+
+if git rev-parse --verify --quiet $current_version
+then
+    echo "Version $current_version already exists in commit:"
+    git --no-pager log -1 $current_version
+    echo
+    echo "Update the version number in setup.py before merging this branch into master."
+    echo "Look at the CONTRIBUTING.md file to learn how the version number should be updated."
+    exit 1
+fi
+
+if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh | grep --quiet CHANGELOG.md
+then
+    echo "CHANGELOG.md has not been modified, while functional changes were made."
+    echo "Explain what you changed before merging this branch into master."
+    echo "Look at the CONTRIBUTING.md file to learn how to write the changelog."
+    exit 2
+fi

.github/publish-git-tag.sh

@@ -0,0 +1,4 @@
+#! /usr/bin/env bash
+
+git tag `python .github/fetch_version.py`  # create a new tag
+git push --tags || true  # update the repository version

.github/workflows/changelog_entry.yaml

@@ -0,0 +1,29 @@
+name: Versioning
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  check-changelog-entry:
+    name: Changelog entry check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Check for changelog entry
+        run: |
+          if [ ! -f "changelog_entry.yaml" ]; then
+            echo "Error: changelog_entry.yaml file is missing."
+            echo "Please add a changelog_entry.yaml file at the root of the repository."
+            exit 1
+          fi
+
+          # Check if the file is empty
+          if [ ! -s "changelog_entry.yaml" ]; then
+            echo "Error: changelog_entry.yaml file is empty."
+            echo "Please add content to the changelog_entry.yaml file."
+            exit 1
+          fi
+
+          echo "Changelog entry found and is not empty."

.github/workflows/main.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  Test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python dependencies
+        run: |
+          uv pip install -e ".[dev]" --system
+      - name: Run tests with coverage
+        run: make test
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          fail_ci_if_error: false
+          verbose: true
+  publish-to-pypi:
+    name: Publish to PyPI
+    needs: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install package
+        run: make install
+      - name: Build package
+        run: python -m build
+      - name: Publish a git tag
+        run: ".github/publish-git-tag.sh || true"
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI }}
+          skip-existing: true

.github/workflows/pr.yaml

@@ -0,0 +1,66 @@
+name: Pull request
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  Lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Install relevant dependencies
+        run: |
+          uv pip install black isort linecheck --system
+      - name: Check code formatting
+        run: make check-format
+
+  Test:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: ["3.11"]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python dependencies
+        run: |
+          uv pip install -e ".[dev]" --system
+      - name: Run tests with coverage
+        run: make test
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          fail_ci_if_error: false
+          verbose: true
+
+  Build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          uv pip install -e ".[dev]" --system
+      - name: Build package
+        run: make build

.github/workflows/versioning.yaml

@@ -0,0 +1,38 @@
+# Workflow that runs on versioning metadata updates.
+
+name: Versioning updates
+on:
+  push:
+      branches:
+        - main
+
+      paths:
+        - changelog_entry.yaml
+        - "!pyproject.toml"
+
+jobs:
+    Versioning:
+        runs-on: ubuntu-latest
+        if: |
+          (!(github.event.head_commit.message == 'Update package version'))
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v4
+              with:
+                repository: ${{ github.event.pull_request.head.repo.full_name }}
+                ref: ${{ github.event.pull_request.head.ref }}
+                token: ${{ secrets.POLICYENGINE_GITHUB }}
+            - name: Setup Python
+              uses: actions/setup-python@v5
+              with:
+                python-version: 3.11
+            - name: Build changelog
+              run: pip install yaml-changelog && make changelog
+            - name: Preview changelog update
+              run: ".github/get-changelog-diff.sh"
+            - name: Update changelog
+              uses: EndBug/add-and-commit@v9
+              with:
+                add: "."
+                message: Update package version
+

CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2025-07-22 00:00:00
+
+### Added
+
+- Initialized project.
+
+
+
+

Makefile

@@ -0,0 +1,34 @@
+test:
+	pytest tests/ --cov=policyengine_data --cov-report=xml --maxfail=0 -v
+
+install:
+	pip install -e ".[dev]"
+
+check-format:
+	linecheck .
+	isort --check-only --profile black src/
+	black . -l 79 --check
+
+format:
+	linecheck . --fix
+	isort --profile black src/
+	black . -l 79
+
+documentation:
+	cd docs && jupyter-book build .
+	python docs/add_plotly_to_book.py docs/_build/html
+
+build:
+	pip install build
+	python -m build
+
+clean:
+	rm -rf dist/ build/ *.egg-info/
+	rm -rf docs/_build/
+
+changelog:
+	build-changelog changelog.yaml --output changelog.yaml --update-last-date --start-from 0.1.0 --append-file changelog_entry.yaml
+	build-changelog changelog.yaml --org PolicyEngine --repo policyengine_data --output CHANGELOG.md --template .github/changelog_template.md
+	bump-version changelog.yaml pyproject.toml
+	rm changelog_entry.yaml || true
+	touch changelog_entry.yaml

changelog.yaml

@@ -0,0 +1,5 @@
+- changes:
+    added:
+    - Initialized project.
+  date: 2025-07-22
+  version: 0.1.0

changelog_entry.yaml

@@ -0,0 +1,6 @@
+- bump: minor
+  changes:
+    changed:
+    - Initialized changelogging.
+    - Added CI workflows and tests.
+    - Set up basic package structure and coverage.

pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "policyengine-data"
+version = "0.1.0"
+description = "A package to create representative microdata for PolicyEngine's policy models."
+readme = "README.md"
+authors = [
+    {name = "PolicyEngine", email = "[email protected]"},
+]
+requires-python = ">=3.11"
+dependencies = [
+    "h5py",
+    "numpy",
+    "pandas",
+    "huggingface_hub",
+    "policyengine-core>=3.6.4",
+    "microdf-python==0.4.4",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-cov",
+    "flake8",
+    "black",
+    "isort",
+    "mypy",
+    "build",
+    "linecheck",
+    "yaml-changelog>=0.1.7",
+]
+
+[tool.setuptools]
+packages = ["policyengine_data"]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"policyengine_data" = ["**/*"]
+
+[tool.isort]
+profile = "black"
+line_length = 79
+
+[tool.black]
+line-length = 79
+target-version = ["py311"]
+
+[project.scripts]
+policyengine-data = "policyengine_data:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

src/policyengine_data/__init__.py

@@ -0,0 +1 @@
+

src/policyengine_data/multi_year_dataset.py

@@ -0,0 +1 @@
+

src/policyengine_data/single_year_dataset.py

@@ -0,0 +1 @@
+