diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1924ae5ff..72a3d49ed 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -10,6 +10,7 @@ on: branches: - main - dev + - polars-dev - bugfix - "release/*" diff --git a/.pylintrc b/.pylintrc index 1f38033b0..724ca7062 100644 --- a/.pylintrc +++ b/.pylintrc @@ -48,4 +48,9 @@ disable= unnecessary-dunder-call, use-dict-literal, invalid-name, - import-outside-toplevel + import-outside-toplevel, + missing-class-docstring, + missing-function-docstring, + fixme, + too-many-locals, + redefined-outer-name diff --git a/ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt index 54d77967b..ed4f23e11 100644 --- a/ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.10-pandas1.5.3-pydantic1.10.11.txt @@ -129,7 +129,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -296,6 +296,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -320,6 +321,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -340,6 +342,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt index 07a45b713..468b2f357 100644 --- a/ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.10-pandas1.5.3-pydantic2.3.0.txt @@ -131,7 +131,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -298,6 +298,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -322,6 +323,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -342,6 +344,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt index b3151a179..e780a5aad 100644 --- a/ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.10-pandas2.0.3-pydantic1.10.11.txt @@ -129,7 +129,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -296,6 +296,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -320,6 +321,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -340,6 +342,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt index 9221e9312..569973a15 100644 --- a/ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.10-pandas2.0.3-pydantic2.3.0.txt @@ -131,7 +131,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -298,6 +298,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -322,6 +323,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -342,6 +344,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt b/ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt index eb4c70447..bdfa9c82a 100644 --- a/ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt +++ b/ci/requirements-py3.10-pandas2.2.0-pydantic1.10.11.txt @@ -292,6 +292,7 @@ pandas==2.2.0 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==2.2.0.240218 @@ -314,6 +315,7 @@ platformdirs==4.2.0 # virtualenv pluggy==1.4.0 # via pytest +polars==0.20.10 pre-commit==3.6.2 prometheus-client==0.20.0 # via jupyter-server @@ -333,6 +335,7 @@ pycparser==2.21 pydantic==1.10.11 # via # fastapi + # polars # ray pygments==2.17.2 # via diff --git a/ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt b/ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt index bba60dea5..93c0137e2 100644 --- a/ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt +++ b/ci/requirements-py3.10-pandas2.2.0-pydantic2.3.0.txt @@ -294,6 +294,7 @@ pandas==2.2.0 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==2.2.0.240218 @@ -316,6 +317,7 @@ platformdirs==4.2.0 # virtualenv pluggy==1.4.0 # via pytest +polars==0.20.10 pre-commit==3.6.2 prometheus-client==0.20.0 # via jupyter-server @@ -335,6 +337,7 @@ pycparser==2.21 pydantic==2.3.0 # via # fastapi + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt index 2e09223f3..b8b68744d 100644 --- a/ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.11-pandas1.5.3-pydantic1.10.11.txt @@ -124,7 +124,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -291,6 +291,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -315,6 +316,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -335,6 +337,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt index aecff2217..bd81a3e31 100644 --- a/ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.11-pandas1.5.3-pydantic2.3.0.txt @@ -126,7 +126,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -293,6 +293,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -317,6 +318,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -337,6 +339,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt index 3925d5cc1..5f1fefa6b 100644 --- a/ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.11-pandas2.0.3-pydantic1.10.11.txt @@ -124,7 +124,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -291,6 +291,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -315,6 +316,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -335,6 +337,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt index f34db8466..d614d251e 100644 --- a/ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.11-pandas2.0.3-pydantic2.3.0.txt @@ -126,7 +126,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -293,6 +293,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -317,6 +318,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -337,6 +339,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt b/ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt index 9f793c39b..ff54b45ba 100644 --- a/ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt +++ b/ci/requirements-py3.11-pandas2.2.0-pydantic1.10.11.txt @@ -287,6 +287,7 @@ pandas==2.2.0 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==2.2.0.240218 @@ -309,6 +310,7 @@ platformdirs==4.2.0 # virtualenv pluggy==1.4.0 # via pytest +polars==0.20.10 pre-commit==3.6.2 prometheus-client==0.20.0 # via jupyter-server @@ -328,6 +330,7 @@ pycparser==2.21 pydantic==1.10.11 # via # fastapi + # polars # ray pygments==2.17.2 # via diff --git a/ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt b/ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt index dcb75a71e..83ad71036 100644 --- a/ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt +++ b/ci/requirements-py3.11-pandas2.2.0-pydantic2.3.0.txt @@ -289,6 +289,7 @@ pandas==2.2.0 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==2.2.0.240218 @@ -311,6 +312,7 @@ platformdirs==4.2.0 # virtualenv pluggy==1.4.0 # via pytest +polars==0.20.10 pre-commit==3.6.2 prometheus-client==0.20.0 # via jupyter-server @@ -330,6 +332,7 @@ pycparser==2.21 pydantic==2.3.0 # via # fastapi + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt index c0d2e01c7..256f420fa 100644 --- a/ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.8-pandas1.5.3-pydantic1.10.11.txt @@ -129,7 +129,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -152,7 +152,7 @@ importlib-metadata==6.8.0 # sphinx # twine # typeguard -importlib-resources==6.1.1 +importlib-resources==6.0.1 # via # jsonschema # jsonschema-specifications @@ -308,6 +308,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -334,6 +335,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -354,6 +356,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt index bdc9ae8ad..097302e29 100644 --- a/ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.8-pandas1.5.3-pydantic2.3.0.txt @@ -131,7 +131,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -154,7 +154,7 @@ importlib-metadata==6.8.0 # sphinx # twine # typeguard -importlib-resources==6.1.1 +importlib-resources==6.0.1 # via # jsonschema # jsonschema-specifications @@ -310,6 +310,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -336,6 +337,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -356,6 +358,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt index f4527705d..1310e666a 100644 --- a/ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.8-pandas2.0.3-pydantic1.10.11.txt @@ -129,7 +129,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -152,7 +152,7 @@ importlib-metadata==6.8.0 # sphinx # twine # typeguard -importlib-resources==6.1.1 +importlib-resources==6.0.1 # via # jsonschema # jsonschema-specifications @@ -308,6 +308,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -334,6 +335,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -354,6 +356,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt index 9839ab36b..8a22a33c5 100644 --- a/ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.8-pandas2.0.3-pydantic2.3.0.txt @@ -131,7 +131,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -154,7 +154,7 @@ importlib-metadata==6.8.0 # sphinx # twine # typeguard -importlib-resources==6.1.1 +importlib-resources==6.0.1 # via # jsonschema # jsonschema-specifications @@ -310,6 +310,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -336,6 +337,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -356,6 +358,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt b/ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt index bf28dd75e..f6f7a1713 100644 --- a/ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.9-pandas1.5.3-pydantic1.10.11.txt @@ -129,7 +129,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -303,6 +303,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -327,6 +328,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -347,6 +349,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt b/ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt index 215381b31..ba49687c6 100644 --- a/ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.9-pandas1.5.3-pydantic2.3.0.txt @@ -131,7 +131,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -305,6 +305,7 @@ pandas==1.5.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -329,6 +330,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -349,6 +351,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt b/ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt index 4201ffe88..46c140b07 100644 --- a/ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt +++ b/ci/requirements-py3.9-pandas2.0.3-pydantic1.10.11.txt @@ -129,7 +129,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -303,6 +303,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -327,6 +328,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -347,6 +349,7 @@ pydantic==1.10.11 # via # fastapi # modin + # polars # ray pygments==2.16.1 # via diff --git a/ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt b/ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt index d5ee05c83..e44433fb6 100644 --- a/ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt +++ b/ci/requirements-py3.9-pandas2.0.3-pydantic2.3.0.txt @@ -131,7 +131,7 @@ grpcio==1.58.0 # via ray h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -305,6 +305,7 @@ pandas==2.0.3 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==1.5.2.221213 @@ -329,6 +330,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server @@ -349,6 +351,7 @@ pydantic==2.3.0 # via # fastapi # modin + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt b/ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt index 23c1bdb74..ae729eaaf 100644 --- a/ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt +++ b/ci/requirements-py3.9-pandas2.2.0-pydantic1.10.11.txt @@ -299,6 +299,7 @@ pandas==2.2.0 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==2.2.0.240218 @@ -321,6 +322,7 @@ platformdirs==4.2.0 # virtualenv pluggy==1.4.0 # via pytest +polars==0.20.10 pre-commit==3.6.2 prometheus-client==0.20.0 # via jupyter-server @@ -340,6 +342,7 @@ pycparser==2.21 pydantic==1.10.11 # via # fastapi + # polars # ray pygments==2.17.2 # via diff --git a/ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt b/ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt index 1b48314d0..9f17413b9 100644 --- a/ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt +++ b/ci/requirements-py3.9-pandas2.2.0-pydantic2.3.0.txt @@ -301,6 +301,7 @@ pandas==2.2.0 # modin # partd # petl + # polars # pyspark # ray pandas-stubs==2.2.0.240218 @@ -323,6 +324,7 @@ platformdirs==4.2.0 # virtualenv pluggy==1.4.0 # via pytest +polars==0.20.10 pre-commit==3.6.2 prometheus-client==0.20.0 # via jupyter-server @@ -342,6 +344,7 @@ pycparser==2.21 pydantic==2.3.0 # via # fastapi + # polars # ray pydantic-core==2.6.3 # via pydantic diff --git a/dev/requirements-3.10.txt b/dev/requirements-3.10.txt index 98fcb8eae..d21b21b61 100644 --- a/dev/requirements-3.10.txt +++ b/dev/requirements-3.10.txt @@ -129,7 +129,7 @@ furo==2022.9.29 geopandas==0.14.0 h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -314,6 +314,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server diff --git a/dev/requirements-3.11.txt b/dev/requirements-3.11.txt index d7b53089d..60983b5d1 100644 --- a/dev/requirements-3.11.txt +++ b/dev/requirements-3.11.txt @@ -124,7 +124,7 @@ furo==2022.9.29 geopandas==0.14.0 h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -309,6 +309,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server diff --git a/dev/requirements-3.8.txt b/dev/requirements-3.8.txt index 1ae5f02a8..3d02bee54 100644 --- a/dev/requirements-3.8.txt +++ b/dev/requirements-3.8.txt @@ -129,7 +129,7 @@ furo==2022.9.29 geopandas==0.13.2 h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -327,6 +327,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server diff --git a/dev/requirements-3.9.txt b/dev/requirements-3.9.txt index 9d856254b..7d3e694f5 100644 --- a/dev/requirements-3.9.txt +++ b/dev/requirements-3.9.txt @@ -129,7 +129,7 @@ furo==2022.9.29 geopandas==0.14.0 h11==0.14.0 # via uvicorn -hypothesis==6.98.9 +hypothesis==6.98.10 identify==2.5.29 # via pre-commit idna==3.4 @@ -321,6 +321,7 @@ platformdirs==3.10.0 # virtualenv pluggy==1.3.0 # via pytest +polars==0.20.10 pre-commit==3.4.0 prometheus-client==0.17.1 # via jupyter-server diff --git a/pandera/api/checks.py b/pandera/api/checks.py index 7f9dfb221..62e2a5fea 100644 --- a/pandera/api/checks.py +++ b/pandera/api/checks.py @@ -14,7 +14,7 @@ from pandera import errors from pandera.api.base.checks import BaseCheck, CheckResult -from pandera.strategies import SearchStrategy +from pandera.strategies.base_strategies import SearchStrategy T = TypeVar("T") diff --git a/pandera/backends/polars/base.py b/pandera/backends/polars/base.py index 2b8b60933..ccc832435 100644 --- a/pandera/backends/polars/base.py +++ b/pandera/backends/polars/base.py @@ -110,7 +110,7 @@ def failure_cases_metadata( if isinstance(err.failure_cases, pl.LazyFrame): raise NotImplementedError - elif isinstance(err.failure_cases, pl.DataFrame): + if isinstance(err.failure_cases, pl.DataFrame): err_failure_cases = err.failure_cases.with_columns( schema_context=pl.lit(err.schema.__class__.__name__), column=pl.lit(err.schema.name), diff --git a/pandera/backends/polars/builtin_checks.py b/pandera/backends/polars/builtin_checks.py index 25398ca97..eb9bcaa89 100644 --- a/pandera/backends/polars/builtin_checks.py +++ b/pandera/backends/polars/builtin_checks.py @@ -1,6 +1,6 @@ """Built-in checks for polars.""" -from typing import Any, TypeVar, Iterable +from typing import Any, TypeVar, Iterable, Union import re import polars as pl @@ -206,7 +206,7 @@ def notin(data: PolarsData, forbidden_values: Iterable) -> pl.LazyFrame: ) def str_matches( data: PolarsData, - pattern: str | re.Pattern, + pattern: Union[str, re.Pattern], ) -> pl.LazyFrame: """Ensure that string values match a regular expression. @@ -290,7 +290,7 @@ def str_length( :param min_value: Minimum length of strings (default: no minimum) :param max_value: Maximum length of strings (default: no maximum) """ - # TODO: consider using len_bytes (faster but returns != n_chars for non ASCII strings + # NOTE: consider using len_bytes (faster but returns != n_chars for non ASCII strings n_chars = pl.col("string_col").str.n_chars() is_in_min = ( n_chars.ge(min_value) if min_value is not None else pl.lit(True) diff --git a/pandera/backends/polars/checks.py b/pandera/backends/polars/checks.py index 2ab92bcb5..c8a7e2b9d 100644 --- a/pandera/backends/polars/checks.py +++ b/pandera/backends/polars/checks.py @@ -38,7 +38,7 @@ def aggregate(self, check_obj: pl.LazyFrame): """Implements aggregation behavior for check object.""" raise NotImplementedError - def preprocess(self, check_obj: pl.LazyFrame, key: str): + def preprocess(self, check_obj: pl.LazyFrame, key: Optional[str]): """Preprocesses a check object before applying the check function.""" # This handles the case of Series validation, which has no other context except # for the index to groupby on. Right now grouping by the index is not allowed. diff --git a/pandera/backends/polars/components.py b/pandera/backends/polars/components.py index a2ee20151..5f4edf3ce 100644 --- a/pandera/backends/polars/components.py +++ b/pandera/backends/polars/components.py @@ -1,19 +1,17 @@ """Validation backend for polars components.""" -from collections import defaultdict from typing import List, Optional, cast import polars as pl from pandera.api.polars.components import Column -from pandera.backends.base import BaseSchemaBackend, CoreCheckResult +from pandera.backends.base import CoreCheckResult from pandera.backends.polars.base import PolarsSchemaBackend from pandera.error_handlers import SchemaErrorHandler from pandera.errors import ( SchemaError, SchemaErrors, SchemaErrorReason, - FailureCaseMetadata, ) diff --git a/pandera/backends/polars/container.py b/pandera/backends/polars/container.py index e8d421354..a4087fc1c 100644 --- a/pandera/backends/polars/container.py +++ b/pandera/backends/polars/container.py @@ -5,7 +5,7 @@ import polars as pl from pandera.api.polars.container import DataFrameSchema -from pandera.backends.base import BaseSchemaBackend, CoreCheckResult +from pandera.backends.base import CoreCheckResult from pandera.backends.polars.base import PolarsSchemaBackend from pandera.error_handlers import SchemaErrorHandler from pandera.errors import ( @@ -28,6 +28,7 @@ def validate( lazy: bool = False, inplace: bool = False, ): + # pylint: disable=no-member error_handler = SchemaErrorHandler(lazy) components = [v for k, v in schema.columns.items()] diff --git a/pandera/engines/polars_engine.py b/pandera/engines/polars_engine.py index 2501baadf..b85684149 100644 --- a/pandera/engines/polars_engine.py +++ b/pandera/engines/polars_engine.py @@ -79,9 +79,7 @@ def check( except TypeError: return False - return self.type == pandera_dtype.type and super().check( - pandera_dtype - ) + return self.type == pandera_dtype.type and super().check(pandera_dtype) def __str__(self) -> str: return str(self.type) @@ -487,7 +485,7 @@ def __belongs_to_categories( return belongs_to_categories def __str__(self): - return f"Category" + return "Category" @Engine.register_dtype(equivalents=["null", pl.Null]) diff --git a/pandera/strategies/base_strategies.py b/pandera/strategies/base_strategies.py index 05c04b4bc..52220e832 100644 --- a/pandera/strategies/base_strategies.py +++ b/pandera/strategies/base_strategies.py @@ -1,6 +1,26 @@ """Base module for `hypothesis`-based strategies for data synthesis.""" -from typing import Callable, Dict, Tuple, Type +from typing import Callable, Dict, Generic, Tuple, Type, TypeVar + + +try: + # pylint: disable=unused-import + from hypothesis.strategies import SearchStrategy, composite +except ImportError: # pragma: no cover + T = TypeVar("T") + + # pylint: disable=too-few-public-methods + class SearchStrategy(Generic[T]): # type: ignore + """placeholder type.""" + + def composite(fn): # type: ignore + """placeholder composite strategy.""" + return fn + + HAS_HYPOTHESIS = False +else: + HAS_HYPOTHESIS = True + # This strategy registry maps (check_name, data_type) -> strategy_function # For example: ("greater_than", pd.DataFrame) -> () diff --git a/pandera/strategies/pandas_strategies.py b/pandera/strategies/pandas_strategies.py index 309c1128c..9fe18d01e 100644 --- a/pandera/strategies/pandas_strategies.py +++ b/pandera/strategies/pandas_strategies.py @@ -20,7 +20,6 @@ Any, Callable, Dict, - Generic, List, Optional, Sequence, @@ -42,29 +41,20 @@ ) from pandera.engines import numpy_engine, pandas_engine from pandera.errors import BaseStrategyOnlyError, SchemaDefinitionError -from pandera.strategies.base_strategies import STRATEGY_DISPATCHER +from pandera.strategies.base_strategies import ( + STRATEGY_DISPATCHER, + HAS_HYPOTHESIS, +) -try: +if HAS_HYPOTHESIS: import hypothesis import hypothesis.extra.numpy as npst import hypothesis.extra.pandas as pdst from hypothesis.internal.filtering import max_len, min_len import hypothesis.strategies as st from hypothesis.strategies import SearchStrategy, composite -except ImportError: # pragma: no cover - T = TypeVar("T") - - # pylint: disable=too-few-public-methods - class SearchStrategy(Generic[T]): # type: ignore - """placeholder type.""" - - def composite(fn): # type: ignore - """placeholder composite strategy.""" - return fn - - HAS_HYPOTHESIS = False else: - HAS_HYPOTHESIS = True + from pandera.strategies.base_strategies import SearchStrategy, composite StrategyFn = Callable[..., SearchStrategy] diff --git a/requirements-docs.txt b/requirements-docs.txt index 6e8e0dc58..9112abecb 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile --no-emit-index-url --output-file=requirements-docs.txt requirements.in @@ -53,9 +53,7 @@ certifi==2023.7.22 # pyproj # requests cffi==1.15.1 - # via - # argon2-cffi-bindings - # cryptography + # via argon2-cffi-bindings cfgv==3.4.0 # via pre-commit chardet==5.2.0 @@ -89,11 +87,7 @@ colorlog==6.7.0 commonmark==0.9.1 # via recommonmark coverage[toml]==7.3.1 - # via - # coverage - # pytest-cov -cryptography==42.0.2 - # via secretstorage + # via pytest-cov dask==2023.9.2 # via # -r requirements.in @@ -180,10 +174,6 @@ isort==5.12.0 # pylint jaraco-classes==3.3.0 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # distributed @@ -480,8 +470,6 @@ rpds-py==0.10.3 # referencing scipy==1.11.2 # via -r requirements.in -secretstorage==3.3.3 - # via keyring send2trash==1.8.2 # via jupyter-server shapely==2.0.1 @@ -558,6 +546,13 @@ text-unidecode==1.3 # via python-slugify tinycss2==1.2.1 # via nbconvert +tomli==2.0.1 + # via + # black + # coverage + # mypy + # pylint + # pytest tomlkit==0.12.1 # via pylint toolz==0.12.0 @@ -585,9 +580,7 @@ twine==4.0.2 typeguard==4.1.5 # via -r requirements.in typer[all]==0.9.0 - # via - # frictionless - # typer + # via frictionless types-click==7.1.8 # via -r requirements.in types-pkg-resources==0.1.3 @@ -605,6 +598,8 @@ types-urllib3==1.26.25.14 typing-extensions==4.8.0 # via # -r requirements.in + # astroid + # black # fastapi # mypy # pydantic @@ -612,6 +607,7 @@ typing-extensions==4.8.0 # typeguard # typer # typing-inspect + # uvicorn typing-inspect==0.9.0 # via -r requirements.in tzdata==2023.3 diff --git a/tests/polars/test_polars_check.py b/tests/polars/test_polars_check.py index 56a8921dd..1e70c6387 100644 --- a/tests/polars/test_polars_check.py +++ b/tests/polars/test_polars_check.py @@ -155,7 +155,9 @@ def check_function( if not skip_fail_case: with pytest.raises(SchemaError): - df = pl.LazyFrame(fail_case_data, orient="row", schema=polars_schema) + df = pl.LazyFrame( + fail_case_data, orient="row", schema=polars_schema + ) schema.validate(df) @@ -243,7 +245,10 @@ def get_data_param(self): self.sample_string_data, "binary" ), }, - {"datatype": Categorical(ordering="physical"), "data": self.sample_string_data}, + { + "datatype": Categorical(ordering="physical"), + "data": self.sample_string_data, + }, { "datatype": Float32, "data": self.convert_data( diff --git a/tests/polars/test_polars_container.py b/tests/polars/test_polars_container.py index 7483ca8f4..0a320887b 100644 --- a/tests/polars/test_polars_container.py +++ b/tests/polars/test_polars_container.py @@ -110,5 +110,4 @@ def test_basic_polars_lazy_dataframe_check_error( query = ldf_basic.pipe(ldf_schema_with_check.validate, lazy=True) validated_df = query.collect() - validated_df == ldf_basic.collect() assert validated_df.frame_equal(ldf_basic.collect()) diff --git a/tests/polars/test_polars_dtypes.py b/tests/polars/test_polars_dtypes.py index 172e03b94..e51ea2378 100644 --- a/tests/polars/test_polars_dtypes.py +++ b/tests/polars/test_polars_dtypes.py @@ -221,7 +221,7 @@ def test_coerce_cast_failed(pl_to_dtype, container): try: pl_to_dtype.coerce(data_container=container) - except Exception as e: + except Exception as e: # pylint: disable=broad-except error = e assert error is not None @@ -230,7 +230,7 @@ def test_coerce_cast_failed(pl_to_dtype, container): try: pl_to_dtype.coerce(data_container=container) - except Exception as e: + except Exception as e: # pylint: disable=broad-except error = e assert error is not None @@ -281,10 +281,22 @@ def test_check_equivalent(dtype): [ (pe.Int8, pe.Int16, False), (pe.Category(categories=["a", "b"]), pe.String, False), - (pe.Decimal(precision=2, scale=1), pe.Decimal(precision=3, scale=2), False), - (pe.Decimal(precision=2, scale=1), pe.Decimal(precision=2, scale=1), True), + ( + pe.Decimal(precision=2, scale=1), + pe.Decimal(precision=3, scale=2), + False, + ), + ( + pe.Decimal(precision=2, scale=1), + pe.Decimal(precision=2, scale=1), + True, + ), (pe.DateTime(), pe.Date, False), - (pe.Category(categories=["a", "b"]), pe.Category(categories=["a", "b"]), True), + ( + pe.Category(categories=["a", "b"]), + pe.Category(categories=["a", "b"]), + True, + ), (pe.DateTime(time_unit="s"), pe.DateTime(time_unit="ns"), False), (pe.DateTime(time_unit="s"), pe.DateTime(time_unit="s"), True), ],