python-data-science-utils/utils/pandas.py at master · LAWTIDEBOY/python-data-science-utils · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import numpy as np
import pandas as pd
import functools
from shapely.geometry import Point
import geopandas as gpd
from fiona.crs import from_epsg

def bin_series(s, bins):
    """
    Sort pandas series into bins, labelling them by each bin's midpoint.

    Parameters
    ----------
    s: pandas Series
    bins: bins argument to pd.cut

    License
    -------
    GNU-GPLv3, (C) A. R.
    (https://github.com/poplarShift/python-data-science-utils)
    """
    labels = np.diff(bins)/2 + bins[:-1]
    return pd.to_numeric(pd.cut(s, bins=bins, labels=labels))

def date_range_with_fix(*args, fixed_date, **kwargs):
    """
    Force pandas' date_range function to contain a specific date
    by snapping to the closest date.

    Parameters
    ----------
    fixed_date : Anything that can be consumed by pd.Timestamp
    Other args: Anything else that goes into pd.date_range

    License
    -------
    GNU-GPLv3, (C) A. R.
    (https://github.com/poplarShift/python-data-science-utils)
    """
    t = pd.date_range(*args, **kwargs)

    diff = t - pd.Timestamp(fixed_date)

    idx = np.abs(diff).argmin()
    offset = diff[idx]

    return t - offset

def with_df_as_numeric(func):
    """
    Decorator to handle a temporary conversion from and back to potentially
    non-numeric columns. Enables e.g. arithmetic operations on datetime columns.

    Usage
    -----
    ```
    with_df_as_numeric(
        lambda d: d.groupby('CYCLE_NUMBER', as_index=False)[['LONGITUDE', 'LATITUDE', 'JULD']].mean(),
    )(df, 'JULD')
    ```

    License
    -------
    GNU-GPLv3, (C) A. R.
    (https://github.com/poplarShift/python-data-science-utils)
    """
    @functools.wraps(func)
    def wrapper(df, columns):
        df = df.copy()
        # convert to numeric
        if isinstance(columns, str):
            columns = [columns]
        dtypes = {c: df[c].dtype for c in columns}
        for c in columns:
            df[c] = np.where(~df[c].isna(), pd.to_numeric(df[c]), np.nan)
        # do actual operation
        df = func(df)
        # convert back
        for c in columns:
            df[c] = df[c].astype(dtypes[c])
        return df
    return wrapper

def df_to_gdf(df, lon='lon', lat='lat'):
    """
    Turn pandas dataframe with latitude, longitude columns into GeoDataFrame with according Point geometry.

    Parameters
    ----------
    df : pandas dataframe
    lon, lat : names of lon, lat columns

    Returns
    -------
    geopandas geodataframe

    License
    -------
    GNU-GPLv3, (C) A. R.
    (https://github.com/poplarShift/python-data-science-utils)
    """
    df = gpd.GeoDataFrame(df).copy()
    df['geometry'] = [Point(x, y) for x, y in zip(df[lon], df[lat])]
    df.crs = from_epsg(4326)
    return df

def pandas_df_to_markdown_table(df):
    """
    Modeled on https://stackoverflow.com/a/33869154

    License
    -------
    GNU-GPLv3, (C) A. R.
    (https://github.com/poplarShift/python-data-science-utils)
    """
    fmt = ['---' for i in range(len(df.columns))]
    df_fmt = pd.DataFrame([fmt], columns=df.columns)
    df_formatted = pd.concat([df_fmt, df])
    return df_formatted.to_csv(sep="|", index=False)