inference/main.py at main · NoaVelner/inference · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
import numpy as np
import pandas as pd
from winioctlcon import METHOD_IN_DIRECT

from algorithm_factory import _build_algs_from_methods, SIGMAS_PER_PARAMETERS_KS, build_params_from_experiment_settings
from experiment_utils import (KS_FIXED_DIST, build_estimator_from_enum, init_log_file, append_rows, metrics_row,
                              true_value_for_stats, MINIMAL_NUMBER_OF_SUBSETS, log_and_plot, rows_from_single_n_results,)
from dists import Beta1D, Trunc1DExpon, TruncGauss1D, TruncGMM2_1D, uniform1D, normal1D, Trunc1DLogNorm, TruncGMM1D
from ci_params_and_config import Methods, ExperimentSettings, AlgorithmConfig


def run_experiments(num_experiments, algs, settings, sort, log_path=None, overwrite=True):
    init_log_file(log_path, overwrite=overwrite)
    n_values, distribution = settings.n_values, settings.dist

    df = []
    for n in n_values:
        print(f"n: {n}")
        results = compute_single_n(n, num_experiments, algs, settings, sort=sort)
        rows = rows_from_single_n_results(results, settings, n)
        append_rows(df, rows, log_path=log_path)

    return pd.DataFrame(df)


def compute_single_n(n, num_experiments, algs, settings, sort):
    params, eps_t, distribution = settings.base_ci_params(n), settings.epsilon, settings.dist
    data = distribution.rvs(size=(num_experiments, n))

    if sort:
        data = np.sort(data, axis=1)

    results = []
    for alg in algs:
        print(alg.name)
        res = alg.run(data=data, settings=settings, n=n, sort=sort, num_experiments=num_experiments, base_params=params)
        results.append(res)

    return results


def empirical_exp_per_eps_and_dist(dist, eps, methods, sbs_mT_func, sbs_description="", n_values=None,
                                   stats='median', num_experiments=1000, json_end="", plot=True, significance=0.1,
                                   save_plot=False, sort=False, overwrite=True, quant=None):

    # quant = None
    # if stats == 'quant': quant = 0.99
    # if stats == 'median': quant = 0.5
    quant = 0.5
    true_values = true_value_for_stats(dist, stats, n_values, quant)

    sigma_by_n = dict(zip(n_values, SIGMAS_PER_PARAMETERS_KS))
    settings = ExperimentSettings(epsilon=eps, domain=dist.get_bounds(), delta=1e-5, significance=significance, quantile=quant,
                                  sigma_per_parameter=sigma_by_n, true_values=true_values, dist=dist, n_values=n_values)

    algs = _build_algs_from_methods(methods=methods, stats=stats, mT_func=sbs_mT_func, sbs_description=sbs_description,
                                    dist=dist)  # todo: remove dist whenever possible, only for invSens debasing

    json_path = f"results/JSONs/eps_{eps}_{dist.name}_n_{n_values[0]}_{n_values[-1]}_{stats}{json_end}.jsonl"
    df_res = run_experiments(num_experiments, algs, settings, sort=sort, log_path=json_path, overwrite=overwrite)
    log_and_plot(df_res, dist, eps, json_path, plot, save_plot, sbs_description, significance, stats)

    return df_res


def ks_experiment(methods, eps=5, end='', significance=0.1):
    np.random.seed(50)
    n_values = np.linspace(2000, 5000, 10, dtype=int)
    # n_values = np.linspace(500, 5000, 10, dtype=int)
    print(n_values)
    mT_function = lambda n: (int(n ** (2 / 3)), MINIMAL_NUMBER_OF_SUBSETS)
    dists = [KS_FIXED_DIST]

    for distribution in dists:
        empirical_exp_per_eps_and_dist(distribution, eps, sbs_mT_func=mT_function, n_values=n_values,
                                       num_experiments=5000, methods=methods, stats='ks', sort=True,
                                       json_end=f"{end}", significance=significance)

def median_experiment(methods, eps=5, end='', overwrite=True, stats='median', significance=0.1):
    np.random.seed(50)

    n_values = np.linspace(500, 5000, 10, dtype=int)
    # n_values = np.array([3000])
    mT_function = lambda n: (int(n ** (2/3)), MINIMAL_NUMBER_OF_SUBSETS)

    gauss = TruncGauss1D(trunc_left=-6, trunc_right=4, sigma=2)
    # beta = Beta1D(0.1, 0.1)
    # beta = Beta1D(0.4, 0.4)
    # beta = Beta1D(1.5, 5)
    # exp = Trunc1DExpon(trunc=5, l=1)
    # gmm = TruncGMM2_1D(mu1=-1.5, sigma1=1, mu2=1.5, sigma2=1, weight=0.5, trunc_left=-5, trunc_right=5)
    # gmm = TruncGMM2_1D(mu1=0.05, sigma1=3, mu2=0.95, sigma2=3, weight=0.45, trunc_left=-5, trunc_right=5)
    # gmm = TruncGMM2_1D(mu1=0, sigma1=1, mu2=6, sigma2=2, weight=0.45, trunc_left=-3, trunc_right=10)
    # gmm = TruncGMM2_1D(mu1=0, sigma1=1, mu2=6, sigma2=1.5, weight=0.42, trunc_left=-3, trunc_right=10)
    # gmm = TruncGMM2_1D(mu1=0, sigma1=0.5, mu2=30, sigma2=5, weight=0.35, trunc_left=-3, trunc_right=35)
    # ln =  Trunc1DLogNorm(trunc=10000, s=3, scale=1.0)
    # unif = uniform1D(low=0, high=1)
    # gmm = TruncGMM2_1D(mu1=0, sigma1=0.1, mu2=3, sigma2=2, weight=0.5, trunc_left=-5, trunc_right=5)
    dists = [gauss]
    print("median pdf:", dists[0].pdf(dists[0].median()))

    dfs = []
    for distribution in dists:
        dfs.append(empirical_exp_per_eps_and_dist(distribution, eps, sbs_mT_func=mT_function, n_values=n_values, sort=True,
                   num_experiments=1000, methods=methods, stats=stats, json_end=f"{end}", significance=significance, overwrite=overwrite))
    return dfs

def debias_experiment(methods, eps=5, end='', overwrite=True, stats='median'):
    np.random.seed(50)
    n_values = np.linspace(250, 1000, 10, dtype=int)
    mT_function = lambda n: (int(n*0.1), MINIMAL_NUMBER_OF_SUBSETS)

    gauss = TruncGauss1D(trunc_left=-6, trunc_right=4, sigma=2)
    beta = Beta1D(0.1, 0.1)
    exp = Trunc1DExpon(trunc=5, l=1)
    # g_new = TruncGauss1D(trunc_left=-6/20, trunc_right=4/20, sigma=2)
    dists = [gauss, beta, exp]
    dists = [gauss]

    dfs = []
    for distribution in dists:
        dfs.append(empirical_exp_per_eps_and_dist(distribution, eps, sbs_mT_func=mT_function, n_values=n_values, sort=True,
                   num_experiments=1000, methods=methods, json_end=f"{end}", significance=0.1, overwrite=overwrite,stats=stats))


if __name__ == '__main__':
    ks_experiment([Methods.NONPRIV_SUB, Methods.PRIVSUB, Methods.PRIV_BOOTSTRAP], end='_alpha_95', eps=5, significance=0.05)