Check license headers across the repository

.github/workflows/license-headers.yaml

@@ -0,0 +1,29 @@
+# Copyright 2026 The PyAthena authors
+#
+# Licensed under the MIT License.
+# See LICENSE or https://opensource.org/licenses/MIT.
+#
+# SPDX-License-Identifier: MIT
+
+name: License Headers
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: true
+      - uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 # v2.81.10
+        with:
+          tool: just
+      - run: just license-headers

AGENTS.md

@@ -47,7 +47,7 @@ Edit the canonical files when maintaining these skills so both agents use the sa
 
 ```bash
 just format   # Auto-fix formatting and imports
-just lint   # Python lint, format check, mypy, and CloudFormation validation
+just lint   # Python lint, format check, mypy, CloudFormation validation, and license headers
 ```
 
 ### Testing

docs/contributing.md

@@ -118,11 +118,14 @@ For a new Python file first published in 2026, use:
 # SPDX-License-Identifier: MIT
 ```
 
-Place the header near the start of the file, before imports or module documentation, while keeping required shebangs, encoding declarations, and front matter in their required positions.
+Place the header at the start of the file, before imports or module documentation, while keeping required shebangs, encoding declarations, and front matter in their required positions.
 Use the equivalent comment syntax for other formats, such as an HTML comment in Markdown.
 For files with YAML front matter, the header may be written as YAML comments inside that front matter.
 Use this placement for GitHub issue templates so the notice belongs to the template metadata rather than the issue body.
 For files that cannot contain comments, and for generated build artifacts or third-party assets, agree on an appropriate attribution location in the issue rather than inserting an invalid header.
+`just lint` and the License Headers workflow check the header with `scripts/check_license_headers.py`.
+The checker exempts symbolic links and empty and binary files, and `scripts/config/license_headers.toml` lists the exempt file suffixes, such as data and lock files, and the other files without the header.
+A new file that is not exempt and has no header, such as third-party material, is added to that list as agreed in the issue.
 
 The short notice and license reference follow [Google's MIT header example](https://opensource.google/documentation/reference/releasing/licenses#mit-header), with an [SPDX identifier](https://spdx.org/licenses/MIT.html) added.
 

justfile

@@ -24,13 +24,17 @@ format:
     uvx ruff@{{RUFF_VERSION}} check --select I --fix .
     uvx ruff@{{RUFF_VERSION}} format .
 
-# Lint, format check, mypy, and CloudFormation validation
-lint:
+# Lint, format check, mypy, CloudFormation validation, and license headers
+lint: license-headers
     uvx ruff@{{RUFF_VERSION}} check .
     uvx ruff@{{RUFF_VERSION}} format --check .
     uv run mypy .
     uv run cfn-lint cloudformation/*.yaml
 
+# Check license headers
+license-headers:
+    uv run python scripts/check_license_headers.py
+
 # Run tests: just test (pyathena|sqla|sqla-async)
 test target="help":
     @just _test-{{ if target =~ "^-" { "help" } else { target } }}

pyproject.toml

@@ -91,6 +91,7 @@ dev = [
     "sphinx-design",
     "types-python-dateutil",
     "cfn-lint>=1",
+    "tomli>=2.0.0; python_version<'3.11'",
 ]
 
 [build-system]

scripts/README.md

@@ -15,7 +15,7 @@ AWS test infrastructure templates are in [cloudformation/](../cloudformation/).
 
 ## Validation
 
-Run the Python and CloudFormation checks, ShellCheck, actionlint, and offline script tests from the repository root:
+Run the Python and CloudFormation checks, the license header check, ShellCheck, actionlint, and offline script tests from the repository root:
 
 ```bash
 mise install shellcheck actionlint
@@ -24,4 +24,4 @@ just scripts
 
 ShellCheck is pinned in `.mise.toml` and checks `scripts/*.sh`.
 The pinned actionlint checks GitHub Actions workflows, including embedded shell commands.
-The script tests under `scripts/tests/` use botocore stubs and require no AWS credentials.
+The script tests under `scripts/tests/` require no AWS credentials; the database sweep tests use botocore stubs.

scripts/check_license_headers.py

@@ -0,0 +1,202 @@
+# Copyright 2026 The PyAthena authors
+#
+# Licensed under the MIT License.
+# See LICENSE or https://opensource.org/licenses/MIT.
+#
+# SPDX-License-Identifier: MIT
+
+"""Check that repository files carry the PyAthena license header."""
+
+# Usage: just license-headers (also run by just lint)
+#
+# Checks tracked and untracked, non-ignored files in the working tree against
+# the header described in docs/contributing.md, with the exemptions in
+# scripts/config/license_headers.toml. Reports missing headers and stale
+# unheaded-files entries; never modifies files.
+
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+HEADER_LINES = (
+    r"Copyright \d{4} The PyAthena authors",
+    "",
+    re.escape("Licensed under the MIT License."),
+    re.escape("See LICENSE or https://opensource.org/licenses/MIT."),
+    "",
+    re.escape("SPDX-License-Identifier: MIT"),
+)
+
+
+def _block(opening: str | None, prefix: str, closing: str | None) -> re.Pattern[str]:
+    blank = re.escape(prefix.rstrip())
+    lines = [f"{re.escape(prefix)}{line}" if line else blank for line in HEADER_LINES]
+    if opening is not None:
+        lines.insert(0, re.escape(opening))
+    if closing is not None:
+        lines.append(re.escape(closing))
+    return re.compile("\n".join(lines) + "(\n|$)")
+
+
+HASH = (_block(None, "# ", None),)
+HTML = _block("<!--", "", "-->")
+JINJA = _block("{#", "", "-#}")
+
+# Comment syntaxes by file suffix; other files use HASH. Add an entry for a new
+# file format with another comment syntax.
+SUFFIX_BLOCKS = {
+    ".css": (_block("/*", " * ", " */"),),
+    ".html": (JINJA, HTML),
+    ".jinja2": (JINJA,),
+    ".jsonc": (_block(None, "// ", None),),
+    ".md": (HTML,),
+    ".rst": (_block("..", "   ", None),),
+}
+
+SHEBANG = re.compile(r"#!.*\n")
+ENCODING = re.compile(r"#.*coding[:=][ \t]*[-\w.]+.*\n")
+# YAML front matter in Markdown files; elsewhere, a leading YAML document marker.
+FRONT_MATTER = "---\n"
+FRONT_MATTER_END = re.compile(r"\n---[ \t]*(\n|$)")
+
+CONFIG = "scripts/config/license_headers.toml"
+
+
+@dataclass(frozen=True)
+class Config:
+    """Exemptions read from CONFIG."""
+
+    exempt_suffixes: frozenset[str]
+    unheaded_files: frozenset[str]
+
+
+def load_config(file: Path) -> Config:
+    """Read the exemptions, rejecting unknown keys and duplicate entries."""
+    with file.open("rb") as f:
+        data = tomllib.load(f)
+    keys = {"exempt-suffixes", "unheaded-files"}
+    if set(data) != keys:
+        raise ValueError(f"{file}: expected keys {sorted(keys)}, found {sorted(data)}")
+    values = {}
+    for key in sorted(keys):
+        items = data[key]
+        if not isinstance(items, list) or not all(isinstance(item, str) for item in items):
+            raise ValueError(f"{file}: {key} must be a list of strings")
+        if len(set(items)) != len(items):
+            raise ValueError(f"{file}: {key} has duplicate entries")
+        values[key] = frozenset(items)
+    return Config(values["exempt-suffixes"], values["unheaded-files"])
+
+
+def _matches(blocks: tuple[re.Pattern[str], ...], text: str, pos: int, end: int) -> bool:
+    return any((match := block.match(text, pos)) and match.end() <= end for block in blocks)
+
+
+def _front_matter_header(text: str, suffix: str, blocks: tuple[re.Pattern[str], ...]) -> bool:
+    if not text.startswith(FRONT_MATTER):
+        return False
+    pos = len(FRONT_MATTER)
+    if suffix != ".md":
+        return _matches(HASH, text, pos, len(text))
+    if not (end := FRONT_MATTER_END.search(text, pos - 1)):
+        return False
+    while pos <= end.start():
+        if _matches(HASH, text, pos, end.start() + 1):
+            return True
+        pos = text.index("\n", pos) + 1
+    return _matches(blocks, text, end.end(), len(text))
+
+
+def has_license_header(text: str, suffix: str) -> bool:
+    """Return whether the header starts a file with the given suffix.
+
+    The header may follow a shebang, an encoding declaration, or a leading YAML
+    document marker. In a Markdown file with YAML front matter, it may be
+    written as YAML comments inside the front matter or follow it.
+    """
+    pos = 0
+    if match := SHEBANG.match(text, pos):
+        pos = match.end()
+    if match := ENCODING.match(text, pos):
+        pos = match.end()
+    blocks = SUFFIX_BLOCKS.get(suffix, HASH)
+    return _matches(blocks, text, pos, len(text)) or _front_matter_header(text, suffix, blocks)
+
+
+def exemption_reason(root: Path, path: str, exempt_suffixes: frozenset[str]) -> str | None:
+    """Return why a file needs no header, or None when it needs one."""
+    file = root / path
+    if file.is_symlink():
+        return "symbolic link"
+    if Path(path).suffix in exempt_suffixes:
+        return "data or generated file"
+    data = file.read_bytes()
+    if b"\0" in data:
+        return "binary file"
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        return "binary file"
+    if not text.strip():
+        return "empty file"
+    return None
+
+
+def check(root: Path, paths: list[str], config: Config) -> list[str]:
+    """Return problems for the given repository-relative paths."""
+    problems = []
+    existing = {path for path in paths if (root / path).is_symlink() or (root / path).is_file()}
+    for path in sorted(existing):
+        file = root / path
+        reason = exemption_reason(root, path, config.exempt_suffixes)
+        headed = reason is None and has_license_header(
+            file.read_text(encoding="utf-8"), Path(path).suffix
+        )
+        if path in config.unheaded_files:
+            if reason is not None:
+                problems.append(f"{path}: listed as unheaded in {CONFIG} but exempt as {reason}")
+            elif headed:
+                problems.append(f"{path}: listed as unheaded in {CONFIG} but has the header")
+        elif reason is None and not headed:
+            problems.append(f"{path}: missing license header")
+    problems.extend(
+        f"{path}: listed as unheaded in {CONFIG} but not found"
+        for path in sorted(config.unheaded_files - existing)
+    )
+    return problems
+
+
+def repository_files(root: Path) -> list[str]:
+    """Return tracked and untracked, non-ignored files."""
+    output = subprocess.run(
+        ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
+        cwd=root,
+        check=True,
+        capture_output=True,
+    ).stdout
+    return [path for path in os.fsdecode(output).split("\0") if path]
+
+
+def main() -> int:
+    output = subprocess.run(
+        ["git", "rev-parse", "--show-toplevel"], check=True, capture_output=True
+    ).stdout
+    root = Path(os.fsdecode(output.removesuffix(b"\n")))
+    problems = check(root, repository_files(root), load_config(root / CONFIG))
+    if not problems:
+        return 0
+    sys.stderr.write("".join(f"{problem}\n" for problem in problems))
+    sys.stderr.write("See docs/contributing.md for the header of new original files.\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/config/license_headers.toml

@@ -0,0 +1,96 @@
+# Copyright 2026 The PyAthena authors
+#
+# Licensed under the MIT License.
+# See LICENSE or https://opensource.org/licenses/MIT.
+#
+# SPDX-License-Identifier: MIT
+
+# Configuration for scripts/check_license_headers.py.
+
+# Formats without comment syntax or with generated content.
+exempt-suffixes = [".csv", ".gz", ".json", ".lock", ".png", ".tsv"]
+
+# Existing files without the header, classified in
+# https://github.com/pyathena-dev/PyAthena/issues/790 and described in NOTICE.
+# New files carry the header; add an entry only as agreed in the issue that
+# proposes the file, and remove entries whose files gain the header or are
+# deleted.
+unheaded-files = [
+    ".github/PULL_REQUEST_TEMPLATE.md",
+    "LICENSE",
+    "NOTICE",
+    "cloudformation/github_actions_oidc.yaml",
+    "docs/aio.md",
+    "docs/arrow.md",
+    "docs/conf.py",
+    "docs/cursor.md",
+    "docs/pandas.md",
+    "docs/polars.md",
+    "docs/s3fs.md",
+    "docs/sqlalchemy.md",
+    "docs/usage.md",
+    "pyathena/__init__.py",
+    "pyathena/aio/arrow/cursor.py",
+    "pyathena/aio/common.py",
+    "pyathena/aio/pandas/cursor.py",
+    "pyathena/aio/polars/cursor.py",
+    "pyathena/aio/result_set.py",
+    "pyathena/arrow/async_cursor.py",
+    "pyathena/arrow/converter.py",
+    "pyathena/arrow/cursor.py",
+    "pyathena/arrow/result_set.py",
+    "pyathena/arrow/util.py",
+    "pyathena/async_cursor.py",
+    "pyathena/common.py",
+    "pyathena/connection.py",
+    "pyathena/converter.py",
+    "pyathena/cursor.py",
+    "pyathena/filesystem/s3.py",
+    "pyathena/filesystem/s3_object.py",
+    "pyathena/formatter.py",
+    "pyathena/model.py",
+    "pyathena/pandas/__init__.py",
+    "pyathena/pandas/async_cursor.py",
+    "pyathena/pandas/converter.py",
+    "pyathena/pandas/cursor.py",
+    "pyathena/pandas/result_set.py",
+    "pyathena/pandas/util.py",
+    "pyathena/parser.py",
+    "pyathena/polars/__init__.py",
+    "pyathena/polars/async_cursor.py",
+    "pyathena/polars/cursor.py",
+    "pyathena/result_set.py",
+    "pyathena/s3fs/async_cursor.py",
+    "pyathena/s3fs/cursor.py",
+    "pyathena/sqlalchemy/array.py",
+    "pyathena/sqlalchemy/base.py",
+    "pyathena/sqlalchemy/compiler.py",
+    "pyathena/sqlalchemy/constants.py",
+    "pyathena/sqlalchemy/temporal.py",
+    "pyathena/sqlalchemy/types.py",
+    "pyathena/util.py",
+    "pyproject.toml",
+    "tests/__init__.py",
+    "tests/pyathena/aio/sqlalchemy/test_base.py",
+    "tests/pyathena/aio/test_cursor.py",
+    "tests/pyathena/arrow/test_async_cursor.py",
+    "tests/pyathena/conftest.py",
+    "tests/pyathena/filesystem/test_s3.py",
+    "tests/pyathena/filesystem/test_s3_async.py",
+    "tests/pyathena/pandas/test_async_cursor.py",
+    "tests/pyathena/pandas/test_cursor.py",
+    "tests/pyathena/pandas/test_util.py",
+    "tests/pyathena/polars/test_async_cursor.py",
+    "tests/pyathena/s3fs/test_cursor.py",
+    "tests/pyathena/sqlalchemy/test_array.py",
+    "tests/pyathena/sqlalchemy/test_base.py",
+    "tests/pyathena/sqlalchemy/test_temporal.py",
+    "tests/pyathena/sqlalchemy/test_types.py",
+    "tests/pyathena/test_async_cursor.py",
+    "tests/pyathena/test_converter.py",
+    "tests/pyathena/test_cursor.py",
+    "tests/pyathena/test_model.py",
+    "tests/pyathena/test_util.py",
+    "tests/resources/queries/create_table.sql.jinja2",
+    "tests/sqlalchemy/test_suite.py",
+]