Type-check package (#229)

* Draft usage of mypy.

* Apply pchs.

* Draft usage of pixi.

* Adapt more integration tests.

* Adapt bigquery integration test task.

* Migrate docs deployment to pixi.

* Update build and publish.

* Run postinstall for readthedocs.

* Adapt liniting to rely on pixi.

* Add pytest as dependency.

* Adapt readthedocs configuration.

* Update development instructions.

* Switch to pixi in helper script.

* Minor adadptations.

* Introduce mypy CI step.

* Add changelog entry.

* Fix usage of condarc file.

* Adapt development.rst.

* Use pixi for mypy job.

* Fix yml syntax.

* Create pixi environment for mypy.

* Remove impala step for debugging.

* Add type packages.

* Comment out impala run for debugging.

* Move flit to host-dependencies.

* Remove redundant dependency.

* Update .github/workflows/ci.yaml

Co-authored-by: Pavel Zwerschke <[email protected]>

* Add color to ci tests.

* Update .github/workflows/ci.yaml

Co-authored-by: Pavel Zwerschke <[email protected]>

* Update .github/workflows/ci.yaml

Co-authored-by: Pavel Zwerschke <[email protected]>

* Remove configurations which were erroneously added.

* Remove configurations which were erroneously added.

* Consistently use ubuntu-latest.

* Update .github/workflows/ci.yaml

Co-authored-by: Pavel Zwerschke <[email protected]>

* Also run unit tests on macos and windows.

* Fix impala test syntax.

* Downgrade sqlalchemy version for impala tests.

* Downgrade sqlalchemy version for impala tests.

* Disable BigQuery tests for now.

---------

Co-authored-by: Pavel Zwerschke <[email protected]>

Author: kklein

.github/workflows/ci.yaml

@@ -29,6 +29,21 @@ jobs:
       - name: pre-commit
         run: pixi run pre-commit-run --color=always --show-diff-on-failure
 
+  mypy-type-checks:
+    name: Mypy Type Checks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+      - name: Set up pixi
+        uses: prefix-dev/[email protected]
+        with:
+          environments: default lint
+      - name: mypy
+        run: |
+          pixi run -e mypy postinstall
+          pixi run -e mypy mypy .
+
   unit-tests:
     name: "unit tests"
     strategy:

CHANGELOG.rst

@@ -11,8 +11,14 @@ Changelog
 ------------------
 
 **New features**
+
 - Added styling for assertion messages. See :ref:`assertion-message-styling` for more information.
 
+**Other changes**
+
+- Provide a ``py.typed`` file.
+
+
 1.8.0 - 2023.06.16
 ------------------
 

pixi.lock

@@ -88,6 +88,13 @@ sqlalchemy = "2.*"
 pytest-cov = "*"
 pytest-xdist = "*"
 
+[feature.mypy.dependencies]
+mypy = "*"
+types-setuptools = "*"
+types-colorama = "*"
+pandas-stubs = "*"
+types-jinja2 = "*"
+
 [feature.lint.dependencies]
 pre-commit = "*"
 black = "*"
@@ -150,3 +157,5 @@ impala-py38 = ["impala", "py38", "sa1", "test"]
 impala-sa1 = ["impala", "sa1", "test"]
 
 lint = { features = ["lint"], no-default-feature = true }
+
+mypy = ["mypy"]

pixi.toml

@@ -46,6 +46,10 @@ skip_glob = '\.eggs/*,\.git/*,\.venv/*,build/*,dist/*'
 default_section = 'THIRDPARTY'
 
 [tool.mypy]
-# Temporary fix.
-no_implicit_optional = false
+no_implicit_optional = true
 allow_empty_bodies = true
+check_untyped_defs = true
+
+[[tool.mypy.overrides]]
+module = ["scipy.*", "impala.*", "pytest_html"]
+ignore_missing_imports = true

pyproject.toml

@@ -117,9 +117,9 @@ def __init__(
         self,
         ref: DataReference,
         *,
-        ref2=None,
-        ref_value: Any = None,
-        name: str = None,
+        ref2: Optional[DataReference] = None,
+        ref_value: Optional[Any] = None,
+        name: Optional[str] = None,
         output_processors: Optional[
             Union[OutputProcessor, List[OutputProcessor]]
         ] = output_processor_limit,

src/datajudge/constraints/base.py

@@ -21,7 +21,9 @@ def retrieve(
 
 
 class ColumnExistence(Column):
-    def __init__(self, ref: DataReference, columns: List[str], name: str = None):
+    def __init__(
+        self, ref: DataReference, columns: List[str], name: Optional[str] = None
+    ):
         super().__init__(ref, ref_value=columns, name=name)
 
     def compare(

src/datajudge/constraints/column.py

@@ -38,10 +38,10 @@ def __init__(
         ref: DataReference,
         use_lower_bound_reference: bool,
         column_type: str,
-        name: str = None,
+        name: Optional[str] = None,
         *,
-        ref2: DataReference = None,
-        min_value: str = None,
+        ref2: Optional[DataReference] = None,
+        min_value: Optional[str] = None,
     ):
         self.format = get_format_from_column_type(column_type)
         self.use_lower_bound_reference = use_lower_bound_reference
@@ -84,10 +84,10 @@ def __init__(
         ref: DataReference,
         use_upper_bound_reference: bool,
         column_type: str,
-        name: str = None,
+        name: Optional[str] = None,
         *,
-        ref2: DataReference = None,
-        max_value: str = None,
+        ref2: Optional[DataReference] = None,
+        max_value: Optional[str] = None,
     ):
         self.format = get_format_from_column_type(column_type)
         self.use_upper_bound_reference = use_upper_bound_reference
@@ -132,7 +132,7 @@ def __init__(
         min_fraction: float,
         lower_bound: str,
         upper_bound: str,
-        name: str = None,
+        name: Optional[str] = None,
     ):
         super().__init__(ref, ref_value=min_fraction, name=name)
         self.lower_bound = lower_bound

src/datajudge/constraints/date.py

@@ -13,10 +13,10 @@ def __init__(
         ref: DataReference,
         aggregation_column: str,
         start_value: int = 0,
-        name: str = None,
+        name: Optional[str] = None,
         *,
         tolerance: float = 0,
-        ref2: DataReference = None,
+        ref2: Optional[DataReference] = None,
     ):
         super().__init__(ref, ref2=ref2, ref_value=object(), name=name)
         self.aggregation_column = aggregation_column

src/datajudge/constraints/groupby.py

@@ -18,7 +18,7 @@ def __init__(
         start_columns: List[str],
         end_columns: List[str],
         max_relative_n_violations: float,
-        name: str = None,
+        name: Optional[str] = None,
     ):
         super().__init__(ref, ref_value=object(), name=name)
         self.key_columns = key_columns
@@ -56,7 +56,9 @@ def retrieve(
         sample_selection, n_violations_selection = self.select(engine, ref)
         with engine.connect() as connection:
             self.sample = connection.execute(sample_selection).first()
-            n_violation_keys = connection.execute(n_violations_selection).scalar()
+            n_violation_keys = int(
+                str(connection.execute(n_violations_selection).scalar())
+            )
 
         selections = [*n_keys_selections, sample_selection, n_violations_selection]
         return (n_violation_keys, n_distinct_key_values), selections
@@ -97,7 +99,7 @@ def select(self, engine: sa.engine.Engine, ref: DataReference):
         return sample_selection, n_violations_selection
 
     @abc.abstractmethod
-    def compare(self, engine: sa.engine.Engine, ref: DataReference):
+    def compare(self, factual: Any, target: Any):
         pass