added statistic folder and config.py

oyetripathi · oyetripathi · commit 84de6320b017 · 2022-06-16T09:04:10.000+05:30
diff --git a/config.py b/config.py
@@ -0,0 +1,105 @@
+from util.time_generation import TimeGeneration
+import os
+
+SERVER_ADDR= 'localhost'   # When running in a real distributed setting, change to the server's IP address
+SERVER_PORT = 51000
+
+dataset_file_path = os.path.join(os.path.dirname(__file__), 'datasets')
+results_file_path = os.path.join(os.path.dirname(__file__), 'results')
+single_run_results_file_path = results_file_path + '/SingleRun.csv'
+multi_run_results_file_path = results_file_path + '/MultipleRuns.csv'
+
+# Model, dataset, and control parameter configurations for MNIST with SVM
+dataset = 'MNIST_ORIG_EVEN_ODD'  # Use for SVM model
+model_name = 'ModelSVMSmooth'
+control_param_phi = 0.025   # Good for MNIST with smooth SVM
+
+# Model, dataset, and control parameter configurations for MNIST with CNN
+# dataset = 'MNIST_ORIG_ALL_LABELS'  # Use for CNN model
+# model_name = 'ModelCNNMnist'
+# control_param_phi = 0.00005   # Good for CNN
+
+# Model, dataset, and control parameter configurations for CIFAR-10 with CNN
+# dataset = 'CIFAR_10'
+# model_name = 'ModelCNNCifar10'
+# control_param_phi = 0.00005   # Good for CNN
+
+n_nodes = 5  # Specifies the total number of clients
+
+moving_average_holding_param = 0.0  # Moving average coefficient to smooth the estimation of beta, delta, and rho
+
+step_size = 0.01
+
+# Setting batch_size equal to total_data makes the system use deterministic gradient descent;
+# Setting batch_size equal < total_data makes the system use stochastic gradient descent.
+# batch_size = 1000  # Value for deterministic gradient descent
+# total_data = 1000  # Value for deterministic gradient descent
+batch_size = 100  # 100  # Value for stochastic gradient descent
+total_data = 60000  # 60000  #Value for stochastic gradient descent
+
+# Choose whether to run a single instance and plot the instantaneous results or
+# run multiple instances and plot average results
+single_run = False
+
+# Choose whether to estimate beta and delta in all runs, including those where tau is not adaptive,
+# this is useful for getting statistics. NOTE: Enabling this may change the communication time when using
+# real-world measurements for resource consumption
+estimate_beta_delta_in_all_runs = False
+
+# If true, the weight corresponding to minimum loss (the loss is estimated if using stochastic gradient descent) is
+# returned. If false, the weight at the end is returned. Setting use_min_loss = True corresponds to the latest
+# theoretical bound for the **DISTRIBUTED** case.
+# For the **CENTRALIZED** case, set use_min_loss = False,
+# because convergence of the final value can be guaranteed in the centralized case.
+use_min_loss = True
+
+# Specifies the number of iterations the client uses the same minibatch, using the same minibatch can reduce
+# the processing time at the client, but may cause a worse model accuracy.
+# We use the same minibatch only when the client receives tau_config = 1
+num_iterations_with_same_minibatch_for_tau_equals_one = 3
+
+# Specifies whether all the data should be read when using stochastic gradient descent.
+# Reading all the data requires much more memory but should avoid slowing down due to file reading.
+read_all_data_for_stochastic = True
+
+MAX_CASE = 4  # Specifies the maximum number of cases, this should be a constant equal to 4
+tau_max = 100  # Specifies the maximum value of tau
+
+# tau_setup = -1 is for the proposed adaptive control algorithm, other values of tau correspond to fixed tau values
+if not single_run:
+    tau_setup_all = [-1, 1, 2, 3, 5, 7, 10, 20, 30, 50, 70, 100]
+    sim_runs = range(0, 2)  #Specifies the simulation seeds in each simulation round
+    case_range = range(0, MAX_CASE)
+else:
+    case_range = [0]   # Change if we want single run with other case, should only have one case
+    tau_setup_all = [-1]   # Should only have one value
+    sim_runs = [0]   # Should only have one value, the value specifies the random seed
+
+
+max_time = 15  # Total time budget in seconds
+
+# If time_gen is None, use actual measured time. Else, use time generated by the TimeGeneration class.
+# time_gen = None
+# time_gen = TimeGeneration(1, 0.0, 1e-10, 0.0, 0.0, 0.0)
+
+multiply_global = 1.0
+multiply_local = 1.0
+
+# These numbers are from measurement on stochastic gradient descent on SVM smooth with MNIST even/odd data.
+time_gen = TimeGeneration(multiply_local * 0.013015156, multiply_local * 0.006946299, 1e-10,
+                          multiply_global * 0.131604348, multiply_global * 0.053873234, 1e-10)
+
+# These numbers are from measurement on CENTRALIZED stochastic gradient descent on SVM smooth with MNIST even/odd data.
+# time_gen = TimeGeneration(multiply_local * 0.009974248, multiply_local * 0.011922926, 1e-10,
+#                           0.0, 0.0, 0.0)
+
+# These numbers are from measurement on deterministic gradient descent on SVM smooth with MNIST even/odd data.
+# time_gen = []
+# time_gen.append(TimeGeneration(multiply_local * 0.020613052, multiply_local * 0.008154439, 1e-10,
+#                                multiply_global * 0.137093837, multiply_global * 0.05548447, 1e-10))  # Case 1
+# time_gen.append(TimeGeneration(multiply_local * 0.021810727, multiply_local * 0.008042984, 1e-10,
+#                                multiply_global * 0.12322071, multiply_global * 0.048079171, 1e-10))  # Case 2
+# time_gen.append(TimeGeneration(multiply_local * 0.095353094, multiply_local * 0.016688657, 1e-10,
+#                                multiply_global * 0.157255906, multiply_global * 0.066722225, 1e-10))  # Case 3
+# time_gen.append(TimeGeneration(multiply_local * 0.022075891, multiply_local * 0.008528005, 1e-10,
+#                                multiply_global * 0.108598094, multiply_global * 0.044627335, 1e-10))  # Case 4
diff --git a/statistic/__init__.py b/statistic/__init__.py
diff --git a/statistic/collect_stat.py b/statistic/collect_stat.py
@@ -0,0 +1,164 @@
+import numpy as np
+import os, sys
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from control_algorithm.adaptive_tau import ControlAlgAdaptiveTauServer
+from config import *
+
+
+class CollectStatistics:
+    def __init__(self, results_file_name=os.path.dirname(__file__)+'/results.csv', is_single_run=False):
+        self.results_file_name = results_file_name
+        self.is_single_run = is_single_run
+
+        if not os.path.exists(os.path.dirname(results_file_name)):
+            os.makedirs(os.path.dirname(results_file_name))
+        if is_single_run:
+            with open(results_file_name, 'a') as f:
+                f.write(
+                    'case,tValue,lossValue,predictionAccuracy,betaAdapt,deltaAdapt,rhoAdapt,tau,it_each_local,it_each_global\n')
+                f.close()
+        else:
+            with open(results_file_name, 'a') as f:
+                f.write(
+                    'Type,Simulation,case,tau_setup,lossValue,predictionAccuracy,avg_tau,stddev_tau,' +
+                    'avg_each_local,stddev_each_local,avg_each_global,stddev_each_global,' +
+                    'avg_betaAdapt,stddev_betaAdapt,' +
+                    'avg_deltaAdapt,stddev_deltaAdapt,avg_rhoAdapt,stddev_rhoAdapt,' +
+                    'total_time_recomputed\n')
+                f.close()
+
+    def init_stat_new_global_round(self):
+
+        if self.is_single_run:
+            self.loss_values = []
+            self.prediction_accuracies = []
+            self.t_values = []
+
+        self.taus = []
+        self.each_locals = []
+        self.each_globals = []
+        self.beta_adapts = []
+        self.delta_adapts = []
+        self.rho_adapts = []
+
+    def init_stat_new_global_round_comp(self):
+
+        if self.is_single_run:
+            self.loss_values = []
+            self.prediction_accuracies = []
+            self.t_values = []
+            self.k = []
+            self.immediate_cost =[]
+
+    def collect_stat_end_local_round(self, case, tau, it_each_local, it_each_global, control_alg, model, train_image,
+                                     train_label, test_image, test_label, w_global, total_time_recomputed):
+
+        self.taus.append(tau)  # Use calculated tau
+        self.each_locals.append(it_each_local)
+        self.each_globals.append(it_each_global)
+
+        if control_alg is not None:
+            # TODO: Should define a getter in control algorithm class and use it here
+            if isinstance(control_alg, ControlAlgAdaptiveTauServer):
+                if control_alg.beta_adapt_mvaverage is not None:
+                    self.beta_adapts.append(control_alg.beta_adapt_mvaverage)
+                elif self.is_single_run:
+                    self.beta_adapts.append(np.nan)
+
+                if control_alg.delta_adapt_mvaverage is not None:
+                    self.delta_adapts.append(control_alg.delta_adapt_mvaverage)
+                elif self.is_single_run:
+                    self.delta_adapts.append(np.nan)
+
+                if control_alg.rho_adapt_mvaverage is not None:
+                    self.rho_adapts.append(control_alg.rho_adapt_mvaverage)
+                elif self.is_single_run:
+                    self.rho_adapts.append(np.nan)
+
+        else:
+            if self.is_single_run:  # When doing a single run, the array needs to align with the timestamp,
+                                    # thus adding an entry on None
+                self.beta_adapts.append(np.nan)
+                self.delta_adapts.append(np.nan)
+                self.rho_adapts.append(np.nan)
+
+        if self.is_single_run:
+
+            loss_value = model.loss(train_image, train_label, w_global)
+            self.loss_values.append(loss_value)
+
+            prediction_accuracy = model.accuracy(test_image, test_label, w_global)
+            self.prediction_accuracies.append(prediction_accuracy)
+
+            self.t_values.append(total_time_recomputed)
+
+            print("***** lossValue: " + str(loss_value))
+
+            with open(self.results_file_name, 'a') as f:
+                f.write(str(case) + ',' + str(total_time_recomputed) + ',' + str(loss_value) + ','
+                        + str(prediction_accuracy) + ','
+                        + str(self.beta_adapts[-1]) + ',' + str(self.delta_adapts[-1]) + ',' + str(self.rho_adapts[-1])
+                        + ',' + str(tau) + ',' + str(it_each_local) + ',' + str(it_each_global) + '\n')
+                f.close()
+
+    def collect_stat_end_local_round_comp(self, case, num_iter, model, train_image, train_label, test_image, test_label,
+                                          w_global, total_time_recomputed, k=None, cost=None):
+        if self.is_single_run:
+            loss_value = model.loss(train_image, train_label, w_global)
+            self.loss_values.append(loss_value)
+
+            prediction_accuracy = model.accuracy(test_image, test_label, w_global)
+            self.prediction_accuracies.append(prediction_accuracy)
+
+            self.t_values.append(total_time_recomputed)
+            self.k.append(k)
+            self.immediate_cost.append(cost)
+
+            print("***** lossValue: " + str(loss_value))
+
+            with open(self.results_file_name, 'a') as f:
+                f.write(str(case) + ',' + str(num_iter) + ',' + str(total_time_recomputed) + ',' + str(loss_value) + ','
+                        + str(prediction_accuracy) + ',' + str(k) + ',' + str(cost) + '\n')
+                f.close()
+
+    def collect_stat_end_global_round(self, sim, case, tau_setup, total_time, model, train_image, train_label,
+                                      test_image, test_label, w_eval, total_time_recomputed):
+        loss_final = model.loss(train_image, train_label, w_eval)
+        accuracy_final = model.accuracy(test_image, test_label, w_eval)
+
+        if not self.is_single_run:
+            taus_array = np.array(self.taus)
+            avg_tau = np.sum(np.dot(taus_array, taus_array)) / np.sum(taus_array)
+            stddev_tau = np.std(taus_array)
+            avg_each_local = np.mean(np.array(self.each_locals))
+            stddev_each_local = np.std(np.array(self.each_locals))
+            avg_each_global = np.mean(np.array(self.each_globals))
+            stddev_each_global = np.std(np.array(self.each_globals))
+            avg_beta_adapt = np.mean(np.array(self.beta_adapts))
+            stddev_beta_adapt = np.std(np.array(self.beta_adapts))
+            avg_delta_adapt = np.mean(np.array(self.delta_adapts))
+            stddev_delta_adapt = np.std(np.array(self.delta_adapts))
+            avg_rho_adapt = np.mean(np.array(self.rho_adapts))
+            stddev_rho_adapt = np.std(np.array(self.rho_adapts))
+
+            if case is None or np.isnan(case):
+                case = None
+                type_str = 'centralized'
+            else:
+                type_str = 'distributed'
+
+            with open(self.results_file_name, 'a') as f:
+                f.write(type_str + ',' + str(sim) + ',' + str(case) + ',' + str(tau_setup) + ','
+                        + str(loss_final) + ',' + str(accuracy_final) + ',' + str(avg_tau) + ',' + str(stddev_tau) + ','
+                        + str(avg_each_local) + ',' + str(stddev_each_local) + ','
+                        + str(avg_each_global) + ',' + str(stddev_each_global) + ','
+                        + str(avg_beta_adapt) + ',' + str(stddev_beta_adapt) + ','
+                        + str(avg_delta_adapt) + ',' + str(stddev_delta_adapt) + ','
+                        + str(avg_rho_adapt) + ',' + str(stddev_rho_adapt) + ','
+                        + str(total_time_recomputed) + ','
+                        + '\n')
+                f.close()
+
+        print('total time', total_time)
+        print('loss value', loss_final)
+        print('accuracy', accuracy_final)
