From 2a63c21526d0c77ae6ed3ab5381325f1a7962e4d Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 08:58:28 -0500 Subject: [PATCH 01/38] header changed to comma-separated for h5py access --- CSVWriter.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/CSVWriter.py b/CSVWriter.py index 681d389..23095fe 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -1,27 +1,21 @@ -import numpy as np - -def CSVWriter(filename,X,Y,R): - - names=X.dtype.names - - colnames="" +#!/usr/bin/env python - for n in names: - colnames+=n+"/f:" +import numpy as np -# colnames+="TrueClass/f:" - for i in xrange(0,Y.shape[1]): - colnames+="true_"+str(i)+"/f:" +def CSVWriter (filename, X, Y, R): + names = X.dtype.names + colnames = [] - for i in xrange(0,R.shape[1]): - colnames+="predict_"+str(i)+"/f:" + for n in names: colnames.append(n) + for i in xrange(0, Y.shape[1]): colnames.append('true_' + str(i)) + for i in xrange(0, R.shape[1]): colnames.append('predict_' + str(i)) f = open(filename, 'w') - f.write(colnames[:-1]+"\n") - - X0=X.view(np.float32).reshape(X.shape + (-1,)) + f.write(','.join(colnames) + '\n') - YI=np.nonzero(Y)[1] - out=np.concatenate((X0,Y,R),axis=1) + X0 = X.view(np.float32).reshape(X.shape + (-1,)) + #YI = np.nonzero(Y)[1] # DB: why is this needed? + out = np.concatenate((X0,Y,R), axis=1) - np.savetxt(f,out,delimiter=',') + np.savetxt(f, out, delimiter=',') + pass From a5c5f02fcfe1c8a30be98f7095ad5bc67067fb7d Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 09:00:38 -0500 Subject: [PATCH 02/38] using lxplus samples --- InputFiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InputFiles.py b/InputFiles.py index e9706c6..e040cf0 100644 --- a/InputFiles.py +++ b/InputFiles.py @@ -17,7 +17,7 @@ for InputData in InputFiles: - InputData="/scratch/data-backup/afarbin/crogan/h5/"+InputData + InputData="/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/"+InputData Files+= [ [InputData, "AA_Gen"], [InputData, "AB_Gen"], From 54d1b39181c7858ea51bf6ba00d8507c91ff231b Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 11:20:20 -0500 Subject: [PATCH 03/38] capitalization fix --- Classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classification.py b/Classification.py index b620365..622d2b4 100644 --- a/Classification.py +++ b/Classification.py @@ -69,5 +69,5 @@ def Build(self): self.Model=model - def Compile(self, Loss="categorical_crossentropy", Optimizer="rmsprop"): + def Compile(self, loss="categorical_crossentropy", Optimizer="rmsprop"): self.Model.compile(loss=loss, optimizer=optimizer,metrics=["accuracy"]) From 39641942d5dff4a75d57e9893bd3a4e981012388 Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 11:22:23 -0500 Subject: [PATCH 04/38] capitalization fix --- Experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experiment.py b/Experiment.py index 09f8ef9..d2f10bc 100644 --- a/Experiment.py +++ b/Experiment.py @@ -94,7 +94,7 @@ print "Compiling the Model... this will take a while." optimizer="sgd" -MyModel.Compile(Loss=loss, Optimizer=optimizer) +MyModel.Compile(loss=loss, Optimizer=optimizer) model=MyModel.Model # Print the summary From b406dab346fc34fcc3efe2005a70820e0f971a46 Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 11:24:21 -0500 Subject: [PATCH 05/38] capitalization fix --- Experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experiment.py b/Experiment.py index d2f10bc..d3fb1aa 100644 --- a/Experiment.py +++ b/Experiment.py @@ -94,7 +94,7 @@ print "Compiling the Model... this will take a while." optimizer="sgd" -MyModel.Compile(loss=loss, Optimizer=optimizer) +MyModel.Compile(loss=loss, optimizer=optimizer) model=MyModel.Model # Print the summary From d869ca40e79dc5f6d176dc4c5aa1cc17bf75ca02 Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 11:24:54 -0500 Subject: [PATCH 06/38] capitalization fix --- Classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classification.py b/Classification.py index 622d2b4..951d2af 100644 --- a/Classification.py +++ b/Classification.py @@ -69,5 +69,5 @@ def Build(self): self.Model=model - def Compile(self, loss="categorical_crossentropy", Optimizer="rmsprop"): + def Compile(self, loss="categorical_crossentropy", optimizer="rmsprop"): self.Model.compile(loss=loss, optimizer=optimizer,metrics=["accuracy"]) From 914524e38b3e4afa829208d2b0cc9e792696e179 Mon Sep 17 00:00:00 2001 From: dbullock1086 Date: Tue, 11 Oct 2016 12:40:48 -0500 Subject: [PATCH 07/38] revert np.float32 to float --- Experiment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Experiment.py b/Experiment.py index d3fb1aa..7b8b2ca 100644 --- a/Experiment.py +++ b/Experiment.py @@ -57,8 +57,8 @@ Train_X_S=Train_X # Now Lets Simplify the structure (Note this requires everything to be a float) -Train_X=Train_X.view(np.float32).reshape(Train_X.shape + (-1,)) -Test_X=Test_X.view(np.float32).reshape(Test_X.shape + (-1,)) +Train_X=Train_X.view(float).reshape(Train_X.shape + (-1,)) +Test_X=Test_X.view(float).reshape(Test_X.shape + (-1,)) # Protect against divide by zero! Train_X=np.nan_to_num(Train_X) From c3d34a38e466950ef2d8e6fd43a9077b3a157335 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Wed, 19 Oct 2016 22:40:12 -0500 Subject: [PATCH 08/38] sample directories --- InputFiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/InputFiles.py b/InputFiles.py index e040cf0..ee8626a 100644 --- a/InputFiles.py +++ b/InputFiles.py @@ -17,7 +17,8 @@ for InputData in InputFiles: - InputData="/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/"+InputData + InputData="/scratch/data-backup/afarbin/crogan/h5/"+InputData + #InputData="/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/"+InputData Files+= [ [InputData, "AA_Gen"], [InputData, "AB_Gen"], From ac30136bed65b7a9f18b4b510a552fc5bd4f213f Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 20 Oct 2016 18:27:22 -0500 Subject: [PATCH 09/38] test CSV output --- CSVWriter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CSVWriter.py b/CSVWriter.py index 23095fe..a1182cd 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -11,7 +11,7 @@ def CSVWriter (filename, X, Y, R): for i in xrange(0, R.shape[1]): colnames.append('predict_' + str(i)) f = open(filename, 'w') - f.write(','.join(colnames) + '\n') + f.write(':'.join(colnames) + '\n') X0 = X.view(np.float32).reshape(X.shape + (-1,)) #YI = np.nonzero(Y)[1] # DB: why is this needed? From ce1c28872e18e76e834826366fba2d9cc60b4ce8 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Sun, 23 Oct 2016 12:40:45 -0500 Subject: [PATCH 10/38] refork --- InputFiles.py | 38 -------------------------------------- InputVars.py | 23 ----------------------- 2 files changed, 61 deletions(-) delete mode 100644 InputFiles.py delete mode 100644 InputVars.py diff --git a/InputFiles.py b/InputFiles.py deleted file mode 100644 index ee8626a..0000000 --- a/InputFiles.py +++ /dev/null @@ -1,38 +0,0 @@ - -# Define the Input Files -InputFiles=["mP1000_mC150_mX100.h5", - "mP1000_mC400_mX100.h5", - "mP1000_mC600_mX100.h5", - "mP1000_mC900_mX100.h5", - "mP1000_mC200_mX100.h5", - "mP1000_mC500_mX100.h5", - "mP1000_mC700_mX100.h5", - "mP1000_mC950_mX100.h5", - "mP1000_mC300_mX100.h5", - "mP1000_mC550_mX100.h5", - "mP1000_mC800_mX100.h5"] -Files=[] - -# Select datasets (formerly TTrees in original ROOT file) - - -for InputData in InputFiles: - InputData="/scratch/data-backup/afarbin/crogan/h5/"+InputData - #InputData="/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/"+InputData - Files+= [ - [InputData, "AA_Gen"], - [InputData, "AB_Gen"], - [InputData, "BA_Gen"], - [InputData, "BB_Gen"] - ] - -Samples=[] - -for F in Files: - if type(F) != str: - name=F[1] - F=F[0] - Samples.append([F,name]) - else: - name=F.split(".")[0].split("/")[-1] - Samples.append([F,name+"_SRAll"]) diff --git a/InputVars.py b/InputVars.py deleted file mode 100644 index 311e70c..0000000 --- a/InputVars.py +++ /dev/null @@ -1,23 +0,0 @@ -# Select Variables To use in training - -# On the JigSaw Dataset Fields will be the following: -# - -FieldGroups = [ - ['mP', 'mC', 'mX', 'METx', 'METy', 'L1_pT', 'L1_M', 'L2_pT','L2_M', 'B1_pT','B1_M', 'B2_pT','B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB','MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], - ['L1_eta', 'L2_eta', 'B1_eta','B2_eta'], - ['L1_phi', 'L2_phi','B1_phi', 'B2_phi'], - ['cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA','cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB','cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB','cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA'], - ['dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB','dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'] ] - -SelectedFields = [ - ['mP', 'mC', 'mX', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'], - - ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'], - - ['METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L2_pT', 'L2_eta', 'L2_phi', 'B1_pT', 'B1_eta', 'B1_phi', 'B2_pT', 'B2_eta', 'B2_phi'], - - ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], - - ['mP', 'mC', 'mX','METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L2_pT', 'L2_eta', 'L2_phi', 'B1_pT', 'B1_eta', 'B1_phi', 'B2_pT', 'B2_eta', 'B2_phi'], -] From 537fa8c0a8b8ee308f8ca4c1ceb4f7afe86766fd Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Sun, 23 Oct 2016 12:47:21 -0500 Subject: [PATCH 11/38] refork --- CSVWriter.py | 4 +-- Classification.py | 4 +-- Experiment.py | 3 +- ScanConfig.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/CSVWriter.py b/CSVWriter.py index a1182cd..10f1dbf 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -11,9 +11,9 @@ def CSVWriter (filename, X, Y, R): for i in xrange(0, R.shape[1]): colnames.append('predict_' + str(i)) f = open(filename, 'w') - f.write(':'.join(colnames) + '\n') + f.write(','.join(colnames) + '\n') - X0 = X.view(np.float32).reshape(X.shape + (-1,)) + X0 = X.view(float).reshape(X.shape + (-1,)) #YI = np.nonzero(Y)[1] # DB: why is this needed? out = np.concatenate((X0,Y,R), axis=1) diff --git a/Classification.py b/Classification.py index 951d2af..07de08f 100644 --- a/Classification.py +++ b/Classification.py @@ -69,5 +69,5 @@ def Build(self): self.Model=model - def Compile(self, loss="categorical_crossentropy", optimizer="rmsprop"): - self.Model.compile(loss=loss, optimizer=optimizer,metrics=["accuracy"]) + def Compile(self, Loss="categorical_crossentropy", Optimizer="rmsprop"): + self.Model.compile(loss=Loss, optimizer=Optimizer,metrics=["accuracy"]) diff --git a/Experiment.py b/Experiment.py index 7b8b2ca..6a9f2ff 100644 --- a/Experiment.py +++ b/Experiment.py @@ -57,6 +57,7 @@ Train_X_S=Train_X # Now Lets Simplify the structure (Note this requires everything to be a float) +# If you get an error that the input size isn't right, try changing float below to float32 or float64 Train_X=Train_X.view(float).reshape(Train_X.shape + (-1,)) Test_X=Test_X.view(float).reshape(Test_X.shape + (-1,)) @@ -94,7 +95,7 @@ print "Compiling the Model... this will take a while." optimizer="sgd" -MyModel.Compile(loss=loss, optimizer=optimizer) +MyModel.Compile(Loss=loss, Optimizer=optimizer) model=MyModel.Model # Print the summary diff --git a/ScanConfig.py b/ScanConfig.py index 1a8731a..aee0a41 100644 --- a/ScanConfig.py +++ b/ScanConfig.py @@ -3,12 +3,80 @@ from DLTools.Permutator import * import sys,argparse +#Previously in InputFiles.py +# Define the Input Files +InputFiles=["mP1000_mC150_mX100.h5", + "mP1000_mC400_mX100.h5", + "mP1000_mC600_mX100.h5", + "mP1000_mC900_mX100.h5", + "mP1000_mC200_mX100.h5", + "mP1000_mC500_mX100.h5", + "mP1000_mC700_mX100.h5", + "mP1000_mC950_mX100.h5", + "mP1000_mC300_mX100.h5", + "mP1000_mC550_mX100.h5", + "mP1000_mC800_mX100.h5"] +Files=[] + +# Select datasets (formerly TTrees in original ROOT file) + + +for InputData in InputFiles: + InputData="/scratch/data-backup/afarbin/crogan/h5/"+InputData + #InputData="/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/"+InputData + Files+= [ + [InputData, "AA_Gen"], + [InputData, "AB_Gen"], + [InputData, "BA_Gen"], + [InputData, "BB_Gen"] + ] + +Samples=[] + +for F in Files: + if type(F) != str: + name=F[1] + F=F[0] + Samples.append([F,name]) + else: + name=F.split(".")[0].split("/")[-1] + Samples.append([F,name+"_SRAll"]) + + +#Previously in InputVars.py + +# Select Variables To use in training + +# On the JigSaw Dataset Fields will be the following: +# + +FieldGroups = [ + ['mP', 'mC', 'mX', 'METx', 'METy', 'L1_pT', 'L1_M', 'L2_pT','L2_M', 'B1_pT','B1_M', 'B2_pT','B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB','MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], + ['L1_eta', 'L2_eta', 'B1_eta','B2_eta'], + ['L1_phi', 'L2_phi','B1_phi', 'B2_phi'], + ['cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA','cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB','cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB','cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA'], + ['dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB','dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'] ] + +SelectedFields = [ + ['mP', 'mC', 'mX', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'], + + ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'], + + ['METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L2_pT', 'L2_eta', 'L2_phi', 'B1_pT', 'B1_eta', 'B1_phi', 'B2_pT', 'B2_eta', 'B2_phi'], + + ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], + + ['mP', 'mC', 'mX','METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L2_pT', 'L2_eta', 'L2_phi', 'B1_pT', 'B1_eta', 'B1_phi', 'B2_pT', 'B2_eta', 'B2_phi'], +] + + + Name="EventClassificationDNN" Config={ "MaxEvents":50000, - "Epochs":1000, + "Epochs":10000, "BatchSize":2048*8, "LearningRate":0.005, From 83ce483d1560a92d306f39ed1e8339b60eb95bde Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Sun, 23 Oct 2016 19:33:17 -0500 Subject: [PATCH 12/38] redundancy --- Experiment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Experiment.py b/Experiment.py index 6a9f2ff..97cd281 100644 --- a/Experiment.py +++ b/Experiment.py @@ -7,7 +7,7 @@ execfile("EventClassificationDNN/Arguments.py") # Now load the Hyperparameters -execfile(ConfigFile) +execfile(ConfigFile) # DB: this "imports" the Samples, FieldGroups, and SelectedFields if "Config" in dir(): for a in Config: @@ -15,13 +15,13 @@ # Load the Data from EventClassificationDNN.MultiClassTools import * -from EventClassificationDNN.InputFiles import Samples +#from EventClassificationDNN.InputFiles import Samples (Train_X, Train_Y), (Test_X, Test_Y), ClassIndex=LoadData(Samples,.1,MaxEvents=MaxEvents) # Select Variables To use in training # To get the field names, just look at Fields=Train_X.dtype.names -from EventClassificationDNN.InputVars import FieldGroups, SelectedFields +#from EventClassificationDNN.InputVars import FieldGroups, SelectedFields # Keep the original data before renomalizing... will use this in output Train_X0=Train_X.copy() From 79c44af51d9e866e0876409b8accf3a3c3283f77 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Wed, 26 Oct 2016 07:06:31 -0500 Subject: [PATCH 13/38] cleanup --- Arguments.py | 66 +++++++-------- CSVWriter.py | 3 +- Experiment.py | 130 ++++++++++++++--------------- ScanConfig.py | 223 +++++++++++++++++++++++++++++--------------------- 4 files changed, 227 insertions(+), 195 deletions(-) diff --git a/Arguments.py b/Arguments.py index 901087d..377dd06 100644 --- a/Arguments.py +++ b/Arguments.py @@ -1,49 +1,45 @@ # Configuration of this job parser = argparse.ArgumentParser() -parser.add_argument('-C', '--config',default="EventClassificationDNN/ScanConfig.py") -parser.add_argument('-I', '--inputdata',default="/data/afarbin/crogan/h5/mP1000_mC200_mX100.h5") -parser.add_argument('-L', '--LoadModel',default=False) -parser.add_argument('--gpu', dest='gpuid', default="") -parser.add_argument('--cpu', action="store_true") -parser.add_argument('--NoTrain', action="store_true") -parser.add_argument('-s',"--hyperparamset", default="0") -parser.add_argument('-v',"--varset", default="0") -parser.add_argument('--NoResults', action="store_false") +parser.add_argument ('-C', '--config', default='EventClassificationDNN/ScanConfig.py') +parser.add_argument ('-I', '--inputdata', default='/data/afarbin/crogan/h5/mP1000_mC200_mX100.h5') +parser.add_argument ('-L', '--LoadModel', default=False) +parser.add_argument ('--gpu', dest='gpuid', default='') +parser.add_argument ('--cpu', action='store_true') +parser.add_argument ('--NoTrain', action='store_true') +parser.add_argument ('-s', '--hyperparamset', type=int, default=0) +parser.add_argument ('-v', '--varset', type=int, default=0) +parser.add_argument ('--NoResults', action='store_false') args = parser.parse_args() -UseGPU=not args.cpu -gpuid=args.gpuid -if args.hyperparamset: - HyperParamSet = int(args.hyperparamset) +UseGPU = not args.cpu +gpuid = args.gpuid +HyperParamSet = args.hyperparamset -if args.varset: - VarSet = int(args.varset) +VarSet = args.varset +print 'Using VarSet:', VarSet - print "Using VarSet: ",VarSet - -ConfigFile=args.config -InputData=args.inputdata - -LoadModel=args.LoadModel +ConfigFile = args.config +InputData = args.inputdata +LoadModel = args.LoadModel # Configuration from PBS: -if "PBS_ARRAYID" in os.environ: - HyperParamSet = int(os.environ["PBS_ARRAYID"]) +if 'PBS_ARRAYID' in os.environ: HyperParamSet = int(os.environ['PBS_ARRAYID']) -if "PBS_QUEUE" in os.environ: - if "cpu" in os.environ["PBS_QUEUE"]: - UseGPU=False - if "gpu" in os.environ["PBS_QUEUE"]: - UseGPU=True - gpuid=int(os.environ["PBS_QUEUE"][3:4]) +if 'PBS_QUEUE' in os.environ: + if 'cpu' in os.environ['PBS_QUEUE']: UseGPU = False + if 'gpu' in os.environ['PBS_QUEUE']: + UseGPU = True + gpuid = int(os.environ['PBS_QUEUE'][3:4]) + pass + pass if UseGPU: - print "Using GPU",gpuid - os.environ['THEANO_FLAGS'] = "mode=FAST_RUN,device=gpu%s,floatX=float32,force_device=True" % (gpuid) -else: - print "Using CPU." + print 'Using GPU', gpuid + os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu%s,floatX=float32,force_device=True' % (gpuid) + pass +else: print 'Using CPU.' -Train= not args.NoTrain +Train = not args.NoTrain -WriteResults= args.NoResults +WriteResults = args.NoResults diff --git a/CSVWriter.py b/CSVWriter.py index 10f1dbf..5ed27f1 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python import numpy as np @@ -14,7 +14,6 @@ def CSVWriter (filename, X, Y, R): f.write(','.join(colnames) + '\n') X0 = X.view(float).reshape(X.shape + (-1,)) - #YI = np.nonzero(Y)[1] # DB: why is this needed? out = np.concatenate((X0,Y,R), axis=1) np.savetxt(f, out, delimiter=',') diff --git a/Experiment.py b/Experiment.py index 97cd281..8ba1d41 100644 --- a/Experiment.py +++ b/Experiment.py @@ -1,75 +1,75 @@ -import sys,os,argparse +import sys, os, argparse import h5py import numpy as np # Parse Arguments -execfile("EventClassificationDNN/Arguments.py") +execfile ('EventClassificationDNN/Arguments.py') # Now load the Hyperparameters -execfile(ConfigFile) # DB: this "imports" the Samples, FieldGroups, and SelectedFields +execfile (ConfigFile) -if "Config" in dir(): - for a in Config: - exec(a+"="+str(Config[a])) +if 'Config' in dir(): + for a in Config: exec (a + '=' + str(Config[a])) + pass # Load the Data from EventClassificationDNN.MultiClassTools import * #from EventClassificationDNN.InputFiles import Samples -(Train_X, Train_Y), (Test_X, Test_Y), ClassIndex=LoadData(Samples,.1,MaxEvents=MaxEvents) - +(Train_X, Train_Y), (Test_X, Test_Y), ClassIndex = LoadData (Samples, .1, MaxEvents=MaxEvents) + # Select Variables To use in training # To get the field names, just look at Fields=Train_X.dtype.names #from EventClassificationDNN.InputVars import FieldGroups, SelectedFields # Keep the original data before renomalizing... will use this in output -Train_X0=Train_X.copy() -Test_X0=Test_X.copy() +Train_X0 = Train_X.copy() +Test_X0 = Test_X.copy() -GroupMins=[0]*len(FieldGroups) -GroupMaxs=[0]*len(FieldGroups) +GroupMins = [0] * len(FieldGroups) +GroupMaxs = [0] * len(FieldGroups) # Normalize Ranges within variable groups e.g. masses, angles (phi, eta, cos separately) -for Fs in xrange(0,len(FieldGroups)): - Mins=[] - Maxs=[] - for varI in xrange(0,len(FieldGroups[Fs])): - Mins+=[np.min(Train_X0[FieldGroups[Fs][varI]])] - Maxs+=[np.max(Train_X0[FieldGroups[Fs][varI]])] - - GroupMins[Fs]=min(Mins) - GroupMaxs[Fs]=max(Maxs) - +for Fs in xrange(len(FieldGroups)): + Mins = [] + Maxs = [] + for varI in xrange(len(FieldGroups[Fs])): + Mins.append ( np.min(Train_X0[FieldGroups[Fs][varI]]) ) + Maxs.append ( np.max(Train_X0[FieldGroups[Fs][varI]]) ) + pass + GroupMins[Fs] = min(Mins) + GroupMaxs[Fs] = max(Maxs) for var in FieldGroups[Fs]: - yy=Train_X[var] - yy[:]= 1./(GroupMaxs[Fs]-GroupMins[Fs]) * (yy-GroupMins[Fs]) + yy = Train_X[var] + yy[:] = 1./(GroupMaxs[Fs]-GroupMins[Fs]) * (yy-GroupMins[Fs]) + yy1 = Test_X[var] + yy1[:] = 1./(GroupMaxs[Fs]-GroupMins[Fs])* (yy1-GroupMins[Fs]) + pass + pass - yy1=Test_X[var] - yy1[:]= 1./(GroupMaxs[Fs]-GroupMins[Fs])* (yy1-GroupMins[Fs]) - -Train_X_N=Train_X +#Train_X_N = Train_X # Keep Only selected Variables -Train_X=Train_X[SelectedFields[VarSet]] -Test_X=Test_X[SelectedFields[VarSet]] +Train_X = Train_X[SelectedFields[VarSet]] +Test_X = Test_X[SelectedFields[VarSet]] -Train_X_S=Train_X +#Train_X_S = Train_X # Now Lets Simplify the structure (Note this requires everything to be a float) # If you get an error that the input size isn't right, try changing float below to float32 or float64 -Train_X=Train_X.view(float).reshape(Train_X.shape + (-1,)) -Test_X=Test_X.view(float).reshape(Test_X.shape + (-1,)) +Train_X = Train_X.view(float).reshape(Train_X.shape + (-1,)) +Test_X = Test_X.view(float).reshape(Test_X.shape + (-1,)) # Protect against divide by zero! -Train_X=np.nan_to_num(Train_X) -Test_X=np.nan_to_num(Test_X) +Train_X = np.nan_to_num(Train_X) +Test_X = np.nan_to_num(Test_X) # Get some Inof -N_Inputs=len(SelectedFields[VarSet]) -N_Classes=np.shape(Train_Y)[1] -print "N Inputs:",N_Inputs -print "N Classes:",N_Classes +N_Inputs = len(SelectedFields[VarSet]) +N_Classes = np.shape(Train_Y)[1] +print 'N Inputs:', N_Inputs +print 'N Classes:', N_Classes # Now Build the Model from DLTools.ModelWrapper import * @@ -78,48 +78,50 @@ from EventClassificationDNN.Classification import FullyConnectedClassification if LoadModel: - print "Loading Model From:",LoadModel - if LoadModel[-1]=="/": - LoadModel=LoadModel[:-1] - Name=os.path.basename(LoadModel) - MyModel=ModelWrapper(Name) - MyModel.InDir=LoadModel + print 'Loading Model From:', LoadModel + if LoadModel[-1]=='/': LoadModel = LoadModel[:-1] + Name = os.path.basename (LoadModel) + MyModel = ModelWrapper (Name) + MyModel.InDir = LoadModel MyModel.Load() + pass else: - MyModel=FullyConnectedClassification(Name,N_Inputs,Width,Depth,N_Classes,WeightInitialization) - MyModel.Build() + Name += '_%s' % VarSet + print 'Model Filename:', Name + MyModel = FullyConnectedClassification (Name, N_Inputs, Width, Depth, N_Classes, WeightInitialization) + MyModel.Build () + pass -MyModel.MetaData["Config"]=Config +MyModel.MetaData['Config'] = Config # Compile the Model -print "Compiling the Model... this will take a while." +print 'Compiling the Model... this will take a while.' -optimizer="sgd" -MyModel.Compile(Loss=loss, Optimizer=optimizer) +optimizer = 'sgd' +MyModel.Compile (Loss=loss, Optimizer=optimizer) -model=MyModel.Model +model = MyModel.Model # Print the summary -model.summary() +model.summary () if Train: - print "Training." - hist=MyModel.Train(Train_X, Train_Y, Epochs, BatchSize) - + print 'Training.' + hist = MyModel.Train(Train_X, Train_Y, Epochs, BatchSize) score = model.evaluate(Test_X, Test_Y , batch_size=BatchSize) - - print "Final Score:",score - - MyModel.MetaData["FinalScore"]=score + print 'Final Score:',score + MyModel.MetaData['FinalScore'] = score + pass # Save -MyModel.Save() +MyModel.Save () # Analysis from EventClassificationDNN.Analysis import MultiClassificationAnalysis -result=MultiClassificationAnalysis(MyModel,Test_X,Test_Y,BatchSize ) +result = MultiClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize) # Dump out the predictions added to the input if WriteResults: - print "Writing Results." + print 'Writing Results.' from EventClassificationDNN.CSVWriter import * - CSVWriter(MyModel.OutDir+"/Result.csv",Test_X0,Test_Y,result) + CSVWriter (MyModel.OutDir+'/Result.csv', Test_X0, Test_Y, result) + pass diff --git a/ScanConfig.py b/ScanConfig.py index aee0a41..7648245 100644 --- a/ScanConfig.py +++ b/ScanConfig.py @@ -5,126 +5,161 @@ #Previously in InputFiles.py # Define the Input Files -InputFiles=["mP1000_mC150_mX100.h5", - "mP1000_mC400_mX100.h5", - "mP1000_mC600_mX100.h5", - "mP1000_mC900_mX100.h5", - "mP1000_mC200_mX100.h5", - "mP1000_mC500_mX100.h5", - "mP1000_mC700_mX100.h5", - "mP1000_mC950_mX100.h5", - "mP1000_mC300_mX100.h5", - "mP1000_mC550_mX100.h5", - "mP1000_mC800_mX100.h5"] -Files=[] +InputFiles = ['mP1000_mC150_mX100.h5', + 'mP1000_mC400_mX100.h5', + 'mP1000_mC600_mX100.h5', + 'mP1000_mC900_mX100.h5', + 'mP1000_mC200_mX100.h5', + 'mP1000_mC500_mX100.h5', + 'mP1000_mC700_mX100.h5', + 'mP1000_mC950_mX100.h5', + 'mP1000_mC300_mX100.h5', + 'mP1000_mC550_mX100.h5', + 'mP1000_mC800_mX100.h5'] # Select datasets (formerly TTrees in original ROOT file) - - +Files = [] for InputData in InputFiles: - InputData="/scratch/data-backup/afarbin/crogan/h5/"+InputData - #InputData="/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/"+InputData - Files+= [ - [InputData, "AA_Gen"], - [InputData, "AB_Gen"], - [InputData, "BA_Gen"], - [InputData, "BB_Gen"] - ] + InputData = '/scratch/data-backup/afarbin/crogan/h5/' + InputData + #InputData = '/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/' + InputData + Files += [ [InputData, 'AA_Gen'], + [InputData, 'AB_Gen'], + [InputData, 'BA_Gen'], + [InputData, 'BB_Gen'] ] + pass -Samples=[] +Samples = [] for F in Files: - if type(F) != str: - name=F[1] - F=F[0] - Samples.append([F,name]) + if type(F) != str: Samples.append(F) else: - name=F.split(".")[0].split("/")[-1] - Samples.append([F,name+"_SRAll"]) - + name = F.split('.')[0].split('/')[-1] + Samples.append ( [F,name+'_SRAll'] ) + pass + pass #Previously in InputVars.py - # Select Variables To use in training - -# On the JigSaw Dataset Fields will be the following: -# - FieldGroups = [ - ['mP', 'mC', 'mX', 'METx', 'METy', 'L1_pT', 'L1_M', 'L2_pT','L2_M', 'B1_pT','B1_M', 'B2_pT','B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB','MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], - ['L1_eta', 'L2_eta', 'B1_eta','B2_eta'], - ['L1_phi', 'L2_phi','B1_phi', 'B2_phi'], - ['cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA','cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB','cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB','cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA'], - ['dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB','dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'] ] + ['mP', 'mC', 'mX', + 'METx', 'METy', + 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', + 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', + 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', + 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta', + 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', + 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', + 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', + 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', + 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', + 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', + 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', + 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', + 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'] +] SelectedFields = [ - ['mP', 'mC', 'mX', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'], - - ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA'], - - ['METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L2_pT', 'L2_eta', 'L2_phi', 'B1_pT', 'B1_eta', 'B1_phi', 'B2_pT', 'B2_eta', 'B2_phi'], - - ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], - - ['mP', 'mC', 'mX','METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L2_pT', 'L2_eta', 'L2_phi', 'B1_pT', 'B1_eta', 'B1_phi', 'B2_pT', 'B2_eta', 'B2_phi'], + # all reconstruction + ['mP', 'mC', 'mX', + 'METx', 'METy', + 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', + 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', + 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', + 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta', + 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', + 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', + 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', + 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', + 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', + 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', + 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', + 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', + 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], + + # detector reconstruction + ['METx', 'METy', + 'L1_pT', 'L1_phi', 'L1_eta', + 'L2_pT', 'L2_phi', 'L2_eta', + 'B1_pT', 'B1_phi', 'B1_eta', + 'B2_pT', 'B2_phi', 'B2_eta'], + + # particle reconstruction + ['METx', 'METy', + 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', + 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', + 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', + 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta'], + + # physics reconstruction + ['mP', 'mC', 'mX', + 'METx', 'METy', + 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', + 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', + 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', + 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta'], + + # Recursive Jigsaw reconstruction + ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', + 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', + 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', + 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', + 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', + 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', + 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', + 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', + 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], ] +Name = 'TrainedModel' +Config = {'MaxEvents': 50000, + 'Epochs': 10000, + 'BatchSize': 2048*8, + 'LearningRate': 0.005, + 'Decay': 0., + 'Momentum': 0., + 'Nesterov': 0., + 'WeightInitialization':"'normal'"} -Name="EventClassificationDNN" - -Config={ - - "MaxEvents":50000, - "Epochs":10000, - "BatchSize":2048*8, - - "LearningRate":0.005, - - "Decay":0., - "Momentum":0., - "Nesterov":0., - - "WeightInitialization":"'normal'" -} - -Params={ "Width":[128], - "Depth":[2], - "loss":[#"'mean_squared_error'", - '"categorical_crossentropy"'], - } +Params = {'Width': [128], + 'Depth': [2], + 'loss': ["'categorical_crossentropy'"]} -PS=Permutator(Params) -Combos=PS.Permutations() +PS = Permutator (Params) +Combos = PS.Permutations () -print "HyperParameter Scan: ", len(Combos), "possible combiniations." +print 'HyperParameter Scan:', len(Combos), 'possible combinations.' -if "HyperParamSet" in dir(): - i=int(HyperParamSet) +if 'HyperParamSet' in dir(): i = int(HyperParamSet) else: # Set Seed based on time random.seed() - i=int(round(len(Combos)*random.random())) - print "Randomly picking HyperParameter Set" - - -if i<0: - print "SetList:" - for j in xrange(0,len(Combos)): - print j,":",Combos[j] + i = int(round( len(Combos)*random.random() )) + print 'Randomly picking HyperParameter Set' + pass +if i < 0: + print 'SetList:' + for j in xrange(len(Combos)): print j, ':', Combos[j] quit() + pass +print 'Picked combination:', i -print "Picked combination: ",i - -for k in Combos[i]: - Config[k]=Combos[i][k] - -for MetaData in Params.keys(): - val=str(Config[MetaData]).replace('"',"") - Name+="_"+val.replace("'","") - -print "Model Filename: ",Name +for k in Combos[i]: Config[k] = Combos[i][k] +#for MetaData in Params: +# val = str(Config[MetaData]).replace ('"','') +# Name += '_' + val +# pass +#print 'Model Filename:', Name From 53e785978fe847d3d9b86089656c868501df695f Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 27 Oct 2016 09:30:30 -0500 Subject: [PATCH 14/38] regroup --- ScanConfig.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/ScanConfig.py b/ScanConfig.py index 7648245..f8558fd 100644 --- a/ScanConfig.py +++ b/ScanConfig.py @@ -40,25 +40,34 @@ #Previously in InputVars.py # Select Variables To use in training -FieldGroups = [ +FieldGroups = [ + # energy scale ['mP', 'mC', 'mX', 'METx', 'METy', - 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', - 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', - 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', - 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta', + 'L1_M', 'L2_M', 'B1_M', 'B2_M', + 'L1_pT', 'L2_pT', 'B1_pT', 'B2_pT' 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB'], + + # transverse angle + ['L1_phi', 'L2_phi', 'B1_phi', 'B2_phi'], + + # pseudorapidity + ['L1_eta', 'L2_eta', 'B1_eta', 'B2_eta'], + + # decay angle + ['cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', - 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', - 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', + 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB'], + + # separation angle + ['dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', - 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'] + 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], ] SelectedFields = [ From 03a2950b26fe2660658328a92be6e478a31130ba Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 27 Oct 2016 10:02:03 -0500 Subject: [PATCH 15/38] trying different grouping --- ScanConfig.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/ScanConfig.py b/ScanConfig.py index f8558fd..eeac784 100644 --- a/ScanConfig.py +++ b/ScanConfig.py @@ -57,21 +57,22 @@ # pseudorapidity ['L1_eta', 'L2_eta', 'B1_eta', 'B2_eta'], - # decay angle - ['cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', - 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', - 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', - 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB'], - - # separation angle - ['dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', - 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', - 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', - 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], + # decay angles by frame + ['cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB'], + ['cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB'], + ['cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB'], + ['cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB'], + ['cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB'], + + # separation angles by frame + ['dphi_PP_Pa_AA', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA', 'dphi_PP_Pa_BB'], + ['dphi_PP_Pb_AA', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA', 'dphi_PP_Pb_BB'], + ['dphi_Pa_Ca_AA', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA', 'dphi_Pa_Ca_BB'], + ['dphi_Pb_Cb_AA', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA', 'dphi_Pb_Cb_BB'], ] SelectedFields = [ - # all reconstruction + # all reconstruction info ['mP', 'mC', 'mX', 'METx', 'METy', 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', @@ -126,6 +127,11 @@ 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], + + # single RJ reconstruction + ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', + 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], ] Name = 'TrainedModel' From cacf5c5ed37d07a8d85bb64ebaa5594e2532e0ed Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 27 Oct 2016 10:07:33 -0500 Subject: [PATCH 16/38] missing comma --- ScanConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ScanConfig.py b/ScanConfig.py index eeac784..26fef30 100644 --- a/ScanConfig.py +++ b/ScanConfig.py @@ -45,7 +45,7 @@ ['mP', 'mC', 'mX', 'METx', 'METy', 'L1_M', 'L2_M', 'B1_M', 'B2_M', - 'L1_pT', 'L2_pT', 'B1_pT', 'B2_pT' + 'L1_pT', 'L2_pT', 'B1_pT', 'B2_pT', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', From 1603b5d75f31adafeae57cd64124d46e08b59ba5 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 27 Oct 2016 10:38:50 -0500 Subject: [PATCH 17/38] trying new style --- Analysis.py | 86 ++++++++++++++++++----------------------------------- 1 file changed, 29 insertions(+), 57 deletions(-) diff --git a/Analysis.py b/Analysis.py index 22c6d98..b3be560 100644 --- a/Analysis.py +++ b/Analysis.py @@ -1,75 +1,47 @@ # Analysis import numpy as np +import matplotlib as mpl +mpl.use('pdf') +import matplotlib.pyplot as plt +from sklearn.metrics import roc_curve, auc -def ClassificationAnalysis(MyModel,Test_X,Test_Y,BatchSize, SignalClassIndex=5): - import matplotlib as mpl - mpl.use('pdf') - import matplotlib.pyplot as plt +def ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, SignalClassIndex=5): + print 'Prediction Analysis.' + result = MyModel.Model.predict (Test_X, batch_size=BatchSize) - from sklearn.metrics import roc_curve, auc + fpr, tpr, _ = roc_curve (Test_Y[:,SignalClassIndex], + result[:,SignalClassIndex]) + roc_auc = auc (fpr, tpr) - print "Prediction Analysis." - result = MyModel.Model.predict(Test_X, batch_size=BatchSize) - - fpr, tpr, _ = roc_curve(Test_Y[:,SignalClassIndex], - result[:,SignalClassIndex]) - roc_auc = auc(fpr, tpr) - lw=2 - plt.plot(fpr,tpr,color='darkorange', - lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) - - print "ROC AUC: ",roc_auc - - plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') - plt.xlim([0.0, 1.0]) - plt.ylim([0.0, 1.05]) - - plt.xlabel('False Positive Rate') - plt.ylabel('True Positive Rate') - - plt.legend(loc="lower right") - - plt.savefig(MyModel.OutDir+"/ROC.pdf") + plt.plot (fpr,tpr,color='darkorange', + lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) + print 'ROC' SignalClassIndex, 'AUC:', roc_auc + pass -mpColors=["blue","green","red","cyan","magenta","yellow","black","white"] -def MultiClassificationAnalysis(MyModel,Test_X,Test_Y,BatchSize): - import matplotlib as mpl - mpl.use('pdf') - import matplotlib.pyplot as plt +mpColors = ['red', 'darkorange', 'lawngreen', 'green', 'lightseagreen', 'cyan', 'royalblue', 'blue', 'blueviolet', 'magenta', 'hotpink'] - from sklearn.metrics import roc_curve, auc - - print "Prediction Analysis." - result = MyModel.Model.predict(Test_X, batch_size=BatchSize) - - NClasses=Test_Y.shape[1] - - for ClassIndex in xrange(0,NClasses): - fpr, tpr, _ = roc_curve(Test_Y[:,ClassIndex], - result[:,ClassIndex]) - roc_auc = auc(fpr, tpr) +def MultiClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize): + print 'Prediction Analysis.' + result = MyModel.Model.predict (Test_X, batch_size=BatchSize) - lw=2 + NClasses = Test_Y.shape[1] - plt.plot(fpr,tpr,color=mpColors[ClassIndex], - lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) + for ClassIndex in xrange(NClasses): + ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, ClassIndex) + pass - print "ROC ",ClassIndex," AUC: ",roc_auc + plt.plot ([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') + plt.xlim ([0.0, 1.0]) + plt.ylim ([0.0, 1.05]) - plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') - plt.xlim([0.0, 1.0]) - plt.ylim([0.0, 1.05]) + plt.xlabel ('False Positive Rate') + plt.ylabel ('True Positive Rate') - plt.xlabel('False Positive Rate') - plt.ylabel('True Positive Rate') - - plt.legend(loc="lower right") + plt.legend (loc='lower right') - - plt.savefig(MyModel.OutDir+"/ROC.pdf") - + plt.savefig (MyModel.OutDir + '/ROC.pdf') return result From 80941b6a5ee6af5805dfa76cc189b5972ebc5ff0 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 14:29:34 -0500 Subject: [PATCH 18/38] simple pedagogy models --- WlnuConfig.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++ ZllConfig.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 WlnuConfig.py create mode 100644 ZllConfig.py diff --git a/WlnuConfig.py b/WlnuConfig.py new file mode 100644 index 0000000..f2b40ef --- /dev/null +++ b/WlnuConfig.py @@ -0,0 +1,85 @@ +import random +import getopt +from DLTools.Permutator import * +import sys,argparse + +#Previously in InputFiles.py +# Define the Input Files +InputFiles = ['Wlnu.h5'] + +# Select datasets (formerly TTrees in original ROOT file) +Files = [] +for InputData in InputFiles: + InputData = '/home/dbullock/samples' + InputData + #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData + Files += [ [InputData, 'Wto1LMET'], + [InputData, '1LMET'] ] + pass + +Samples = [] + +for F in Files: Samples.append(F) + +#Previously in InputVars.py +# Select Variables To use in training +FieldGroups = [ + # energy scale + ['L_pT', 'L_E', + 'MET_mag'] + + # transverse angle + ['L_phi', 'MET_phi'], + + # pseudorapidity + ['L_eta'], +] + +SelectedFields = [ + ['L_pT', 'L_eta', 'L_phi', 'L_E', + 'MET_mag', 'MET_phi'] +] + +Name = 'WlnuModel' + +Config = {'MaxEvents': 50000, + 'Epochs': 10000, + 'BatchSize': 2048*8, + 'LearningRate': 0.005, + 'Decay': 0., + 'Momentum': 0., + 'Nesterov': 0., + 'WeightInitialization':"'normal'"} + +Params = {'Width': [128], + 'Depth': [2], + 'loss': ["'categorical_crossentropy'"]} + +PS = Permutator (Params) +Combos = PS.Permutations () + +print 'HyperParameter Scan:', len(Combos), 'possible combinations.' + +if 'HyperParamSet' in dir(): i = int(HyperParamSet) +else: + # Set Seed based on time + random.seed() + i = int(round( len(Combos)*random.random() )) + print 'Randomly picking HyperParameter Set' + pass + +if i < 0: + print 'SetList:' + for j in xrange(len(Combos)): print j, ':', Combos[j] + quit() + pass + +print 'Picked combination:', i + +for k in Combos[i]: Config[k] = Combos[i][k] + +#for MetaData in Params: +# val = str(Config[MetaData]).replace ('"','') +# Name += '_' + val +# pass + +#print 'Model Filename:', Name diff --git a/ZllConfig.py b/ZllConfig.py new file mode 100644 index 0000000..6cb6509 --- /dev/null +++ b/ZllConfig.py @@ -0,0 +1,85 @@ +import random +import getopt +from DLTools.Permutator import * +import sys,argparse + +#Previously in InputFiles.py +# Define the Input Files +InputFiles = ['Zll.h5'] + +# Select datasets (formerly TTrees in original ROOT file) +Files = [] +for InputData in InputFiles: + InputData = '/home/dbullock/samples' + InputData + #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData + Files += [ [InputData, 'Zto2LOS'], + [InputData, '2LOS'] ] + pass + +Samples = [] + +for F in Files: Samples.append(F) + +#Previously in InputVars.py +# Select Variables To use in training +FieldGroups = [ + # energy scale + ['LP_pT', 'LP_E', + 'LM_pT', 'LM_E'] + + # transverse angle + ['LP_phi', 'LM_phi'], + + # pseudorapidity + ['LP_eta', 'LM_eta'], +] + +SelectedFields = [ + ['LP_pT', 'LP_eta', 'LP_phi', 'LP_E', + 'LM_pT', 'LM_eta', 'LM_phi', 'LM_E'] +] + +Name = 'ZllModel' + +Config = {'MaxEvents': 50000, + 'Epochs': 10000, + 'BatchSize': 2048*8, + 'LearningRate': 0.005, + 'Decay': 0., + 'Momentum': 0., + 'Nesterov': 0., + 'WeightInitialization':"'normal'"} + +Params = {'Width': [128], + 'Depth': [2], + 'loss': ["'categorical_crossentropy'"]} + +PS = Permutator (Params) +Combos = PS.Permutations () + +print 'HyperParameter Scan:', len(Combos), 'possible combinations.' + +if 'HyperParamSet' in dir(): i = int(HyperParamSet) +else: + # Set Seed based on time + random.seed() + i = int(round( len(Combos)*random.random() )) + print 'Randomly picking HyperParameter Set' + pass + +if i < 0: + print 'SetList:' + for j in xrange(len(Combos)): print j, ':', Combos[j] + quit() + pass + +print 'Picked combination:', i + +for k in Combos[i]: Config[k] = Combos[i][k] + +#for MetaData in Params: +# val = str(Config[MetaData]).replace ('"','') +# Name += '_' + val +# pass + +#print 'Model Filename:', Name From 0b1605b2915e26a18f643fd5db9fe530e331c65c Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 14:42:19 -0500 Subject: [PATCH 19/38] name fix --- WlnuConfig.py | 2 +- ZllConfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/WlnuConfig.py b/WlnuConfig.py index f2b40ef..5d204a4 100644 --- a/WlnuConfig.py +++ b/WlnuConfig.py @@ -13,7 +13,7 @@ InputData = '/home/dbullock/samples' + InputData #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData Files += [ [InputData, 'Wto1LMET'], - [InputData, '1LMET'] ] + [InputData, 'Rndm1LMET'] ] pass Samples = [] diff --git a/ZllConfig.py b/ZllConfig.py index 6cb6509..47ce3da 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -13,7 +13,7 @@ InputData = '/home/dbullock/samples' + InputData #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData Files += [ [InputData, 'Zto2LOS'], - [InputData, '2LOS'] ] + [InputData, 'Rndm2LOS'] ] pass Samples = [] From 552a617e09a68621262d9a9f900522996e60de10 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 14:54:50 -0500 Subject: [PATCH 20/38] missing comma --- WlnuConfig.py | 2 +- ZllConfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/WlnuConfig.py b/WlnuConfig.py index 5d204a4..b9acb49 100644 --- a/WlnuConfig.py +++ b/WlnuConfig.py @@ -25,7 +25,7 @@ FieldGroups = [ # energy scale ['L_pT', 'L_E', - 'MET_mag'] + 'MET_mag'], # transverse angle ['L_phi', 'MET_phi'], diff --git a/ZllConfig.py b/ZllConfig.py index 47ce3da..04e41b0 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -25,7 +25,7 @@ FieldGroups = [ # energy scale ['LP_pT', 'LP_E', - 'LM_pT', 'LM_E'] + 'LM_pT', 'LM_E'], # transverse angle ['LP_phi', 'LM_phi'], From 735212b41b22bbd9a8630277cffcb8b5778410e3 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 14:56:30 -0500 Subject: [PATCH 21/38] missing slash --- WlnuConfig.py | 2 +- ZllConfig.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/WlnuConfig.py b/WlnuConfig.py index b9acb49..ea8634d 100644 --- a/WlnuConfig.py +++ b/WlnuConfig.py @@ -10,7 +10,7 @@ # Select datasets (formerly TTrees in original ROOT file) Files = [] for InputData in InputFiles: - InputData = '/home/dbullock/samples' + InputData + InputData = '/home/dbullock/samples/' + InputData #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData Files += [ [InputData, 'Wto1LMET'], [InputData, 'Rndm1LMET'] ] diff --git a/ZllConfig.py b/ZllConfig.py index 04e41b0..212aa92 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -10,7 +10,7 @@ # Select datasets (formerly TTrees in original ROOT file) Files = [] for InputData in InputFiles: - InputData = '/home/dbullock/samples' + InputData + InputData = '/home/dbullock/samples/' + InputData #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData Files += [ [InputData, 'Zto2LOS'], [InputData, 'Rndm2LOS'] ] From 06d41ff4521a455f53e206a73d6f7a020c72b27e Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 15:28:13 -0500 Subject: [PATCH 22/38] missing comma --- Analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Analysis.py b/Analysis.py index b3be560..4577cc2 100644 --- a/Analysis.py +++ b/Analysis.py @@ -18,7 +18,7 @@ def ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, SignalClassIndex plt.plot (fpr,tpr,color='darkorange', lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) - print 'ROC' SignalClassIndex, 'AUC:', roc_auc + print 'ROC', SignalClassIndex, 'AUC:', roc_auc pass From d445c0123e0b29d311365fe1c6be2bff408770d3 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 15:38:00 -0500 Subject: [PATCH 23/38] missing variable --- Analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Analysis.py b/Analysis.py index 4577cc2..151d7c9 100644 --- a/Analysis.py +++ b/Analysis.py @@ -6,7 +6,6 @@ from sklearn.metrics import roc_curve, auc def ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, SignalClassIndex=5): - print 'Prediction Analysis.' result = MyModel.Model.predict (Test_X, batch_size=BatchSize) fpr, tpr, _ = roc_curve (Test_Y[:,SignalClassIndex], @@ -34,6 +33,8 @@ def MultiClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize): ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, ClassIndex) pass + lw=2 + plt.plot ([0, 1], [0, 1], color='navy', lw=lw, linestyle='--') plt.xlim ([0.0, 1.0]) plt.ylim ([0.0, 1.05]) From 9a8f437b842927c60472b8cdd149cecbfd9bdead Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 1 Nov 2016 19:35:41 -0500 Subject: [PATCH 24/38] reorder for color --- Analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Analysis.py b/Analysis.py index 151d7c9..1b4fc86 100644 --- a/Analysis.py +++ b/Analysis.py @@ -5,6 +5,8 @@ import matplotlib.pyplot as plt from sklearn.metrics import roc_curve, auc +mpColors = ['red', 'darkorange', 'lawngreen', 'green', 'lightseagreen', 'cyan', 'royalblue', 'blue', 'blueviolet', 'magenta', 'hotpink'] + def ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, SignalClassIndex=5): result = MyModel.Model.predict (Test_X, batch_size=BatchSize) @@ -14,15 +16,13 @@ def ClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize, SignalClassIndex lw=2 - plt.plot (fpr,tpr,color='darkorange', + plt.plot (fpr,tpr,color=mpColors[SignalClassIndex], lw=lw, label='ROC curve (area = %0.2f)' % roc_auc) print 'ROC', SignalClassIndex, 'AUC:', roc_auc pass -mpColors = ['red', 'darkorange', 'lawngreen', 'green', 'lightseagreen', 'cyan', 'royalblue', 'blue', 'blueviolet', 'magenta', 'hotpink'] - def MultiClassificationAnalysis (MyModel, Test_X, Test_Y, BatchSize): print 'Prediction Analysis.' result = MyModel.Model.predict (Test_X, batch_size=BatchSize) From 290d2b1ff7650d24ee6a9a72ae2fcfb644726119 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 10 Nov 2016 15:07:31 -0600 Subject: [PATCH 25/38] new configs --- Experiment.py | 1 + ScanConfig.py => SUSYConfig.py | 133 ++++++++++++++++----------------- WlnuConfig.py | 85 --------------------- ZllConfig.py | 17 ++--- 4 files changed, 72 insertions(+), 164 deletions(-) rename ScanConfig.py => SUSYConfig.py (60%) delete mode 100644 WlnuConfig.py diff --git a/Experiment.py b/Experiment.py index 8ba1d41..33cc9dc 100644 --- a/Experiment.py +++ b/Experiment.py @@ -67,6 +67,7 @@ # Get some Inof N_Inputs = len(SelectedFields[VarSet]) +Width = sum ( [ len(FieldGroups[i]) for i in xrange(len(FieldGroups)) ] ) N_Classes = np.shape(Train_Y)[1] print 'N Inputs:', N_Inputs print 'N Classes:', N_Classes diff --git a/ScanConfig.py b/SUSYConfig.py similarity index 60% rename from ScanConfig.py rename to SUSYConfig.py index 26fef30..1bddba1 100644 --- a/ScanConfig.py +++ b/SUSYConfig.py @@ -20,8 +20,7 @@ # Select datasets (formerly TTrees in original ROOT file) Files = [] for InputData in InputFiles: - InputData = '/scratch/data-backup/afarbin/crogan/h5/' + InputData - #InputData = '/afs/cern.ch/work/a/afarbin/public/RestFrames-llbbMET/' + InputData + InputData = os.getenv('SampleDir') + '/' + InputData Files += [ [InputData, 'AA_Gen'], [InputData, 'AB_Gen'], [InputData, 'BA_Gen'], @@ -30,55 +29,53 @@ Samples = [] -for F in Files: - if type(F) != str: Samples.append(F) - else: - name = F.split('.')[0].split('/')[-1] - Samples.append ( [F,name+'_SRAll'] ) - pass - pass +for F in Files: Samples.append(F) #Previously in InputVars.py # Select Variables To use in training + +# used for scaling FieldGroups = [ # energy scale ['mP', 'mC', 'mX', 'METx', 'METy', - 'L1_M', 'L2_M', 'B1_M', 'B2_M', - 'L1_pT', 'L2_pT', 'B1_pT', 'B2_pT', + 'L1_pT', 'L1_M', + 'L2_pT','L2_M', + 'B1_pT','B1_M', + 'B2_pT','B2_M', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', - 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB'], - - # transverse angle - ['L1_phi', 'L2_phi', 'B1_phi', 'B2_phi'], - - # pseudorapidity - ['L1_eta', 'L2_eta', 'B1_eta', 'B2_eta'], - - # decay angles by frame - ['cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB'], - ['cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB'], - ['cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB'], - ['cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB'], - ['cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB'], - - # separation angles by frame - ['dphi_PP_Pa_AA', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA', 'dphi_PP_Pa_BB'], - ['dphi_PP_Pb_AA', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA', 'dphi_PP_Pb_BB'], - ['dphi_Pa_Ca_AA', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA', 'dphi_Pa_Ca_BB'], - ['dphi_Pb_Cb_AA', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA', 'dphi_Pb_Cb_BB'], + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], + + # pseudo rapidity + ['L1_eta', 'L2_eta', 'B1_eta','B2_eta'], + + # azimutahl angle + ['L1_phi', 'L2_phi','B1_phi', 'B2_phi'], + + # decay angles (by frame) + ['cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB'] + ['cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB'] + ['cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB'] + ['cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB'] + ['cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB'] + + # separation angles (by frame) + ['dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA'] + ['dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA'] + ['dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA'] + ['dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA'] ] SelectedFields = [ - # all reconstruction info + # all observables ['mP', 'mC', 'mX', + 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', + 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', + 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', + 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', 'METx', 'METy', - 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', - 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', - 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', - 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', @@ -93,28 +90,21 @@ 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], # detector reconstruction - ['METx', 'METy', - 'L1_pT', 'L1_phi', 'L1_eta', - 'L2_pT', 'L2_phi', 'L2_eta', - 'B1_pT', 'B1_phi', 'B1_eta', - 'B2_pT', 'B2_phi', 'B2_eta'], - - # particle reconstruction - ['METx', 'METy', - 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', - 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', - 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', - 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta'], + ['L1_pT', 'L1_eta', 'L1_phi', 'L1_M', + 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', + 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', + 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', + 'METx', 'METy'], # physics reconstruction ['mP', 'mC', 'mX', - 'METx', 'METy', - 'L1_M', 'L1_pT', 'L1_phi', 'L1_eta', - 'L2_M', 'L2_pT', 'L2_phi', 'L2_eta', - 'B1_M', 'B1_pT', 'B1_phi', 'B1_eta', - 'B2_M', 'B2_pT', 'B2_phi', 'B2_eta'], + 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', + 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', + 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', + 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', + 'METx', 'METy'], - # Recursive Jigsaw reconstruction + # recursive jigsaw reconstruction ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', @@ -128,13 +118,28 @@ 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], - # single RJ reconstruction + # single topology reconstruction ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], + + # adjusted recursive jigsaw reconstruction + ['mP', 'mC', 'mX', + 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', + 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', + 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', + 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', + 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', + 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', + 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', + 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', + 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], ] -Name = 'TrainedModel' +Name = 'SUSYModel' Config = {'MaxEvents': 50000, 'Epochs': 10000, @@ -145,9 +150,10 @@ 'Nesterov': 0., 'WeightInitialization':"'normal'"} -Params = {'Width': [128], - 'Depth': [2], - 'loss': ["'categorical_crossentropy'"]} +#Params = {'Width': [128], +# 'Depth': [2], +# 'loss': ["'categorical_crossentropy'"]} +Params = {'Depth': [2]} PS = Permutator (Params) Combos = PS.Permutations () @@ -171,10 +177,3 @@ print 'Picked combination:', i for k in Combos[i]: Config[k] = Combos[i][k] - -#for MetaData in Params: -# val = str(Config[MetaData]).replace ('"','') -# Name += '_' + val -# pass - -#print 'Model Filename:', Name diff --git a/WlnuConfig.py b/WlnuConfig.py deleted file mode 100644 index ea8634d..0000000 --- a/WlnuConfig.py +++ /dev/null @@ -1,85 +0,0 @@ -import random -import getopt -from DLTools.Permutator import * -import sys,argparse - -#Previously in InputFiles.py -# Define the Input Files -InputFiles = ['Wlnu.h5'] - -# Select datasets (formerly TTrees in original ROOT file) -Files = [] -for InputData in InputFiles: - InputData = '/home/dbullock/samples/' + InputData - #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData - Files += [ [InputData, 'Wto1LMET'], - [InputData, 'Rndm1LMET'] ] - pass - -Samples = [] - -for F in Files: Samples.append(F) - -#Previously in InputVars.py -# Select Variables To use in training -FieldGroups = [ - # energy scale - ['L_pT', 'L_E', - 'MET_mag'], - - # transverse angle - ['L_phi', 'MET_phi'], - - # pseudorapidity - ['L_eta'], -] - -SelectedFields = [ - ['L_pT', 'L_eta', 'L_phi', 'L_E', - 'MET_mag', 'MET_phi'] -] - -Name = 'WlnuModel' - -Config = {'MaxEvents': 50000, - 'Epochs': 10000, - 'BatchSize': 2048*8, - 'LearningRate': 0.005, - 'Decay': 0., - 'Momentum': 0., - 'Nesterov': 0., - 'WeightInitialization':"'normal'"} - -Params = {'Width': [128], - 'Depth': [2], - 'loss': ["'categorical_crossentropy'"]} - -PS = Permutator (Params) -Combos = PS.Permutations () - -print 'HyperParameter Scan:', len(Combos), 'possible combinations.' - -if 'HyperParamSet' in dir(): i = int(HyperParamSet) -else: - # Set Seed based on time - random.seed() - i = int(round( len(Combos)*random.random() )) - print 'Randomly picking HyperParameter Set' - pass - -if i < 0: - print 'SetList:' - for j in xrange(len(Combos)): print j, ':', Combos[j] - quit() - pass - -print 'Picked combination:', i - -for k in Combos[i]: Config[k] = Combos[i][k] - -#for MetaData in Params: -# val = str(Config[MetaData]).replace ('"','') -# Name += '_' + val -# pass - -#print 'Model Filename:', Name diff --git a/ZllConfig.py b/ZllConfig.py index 212aa92..41cb23a 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -10,8 +10,7 @@ # Select datasets (formerly TTrees in original ROOT file) Files = [] for InputData in InputFiles: - InputData = '/home/dbullock/samples/' + InputData - #InputData = '/afs/cern.ch/work/d/dbullock/public/samples/dnn-tutorial/' + InputData + InputData = os.getenv('SampleDir') + '/' + InputData Files += [ [InputData, 'Zto2LOS'], [InputData, 'Rndm2LOS'] ] pass @@ -50,9 +49,10 @@ 'Nesterov': 0., 'WeightInitialization':"'normal'"} -Params = {'Width': [128], - 'Depth': [2], - 'loss': ["'categorical_crossentropy'"]} +#Params = {'Width': [8], +# 'Depth': [2], +# 'loss': ["'categorical_crossentropy'"]} +Params = {'Depth': [2]} PS = Permutator (Params) Combos = PS.Permutations () @@ -76,10 +76,3 @@ print 'Picked combination:', i for k in Combos[i]: Config[k] = Combos[i][k] - -#for MetaData in Params: -# val = str(Config[MetaData]).replace ('"','') -# Name += '_' + val -# pass - -#print 'Model Filename:', Name From df6a356d8983cb076da0a8be760fe537034d7612 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 10 Nov 2016 21:11:18 -0600 Subject: [PATCH 26/38] new tutorial test --- CSVWriter.py | 2 +- Experiment.py | 5 +++-- SUSYConfig.py | 6 ++---- ZllConfig.py | 6 ++---- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/CSVWriter.py b/CSVWriter.py index 5ed27f1..31733a2 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -13,7 +13,7 @@ def CSVWriter (filename, X, Y, R): f = open(filename, 'w') f.write(','.join(colnames) + '\n') - X0 = X.view(float).reshape(X.shape + (-1,)) + X0 = X.view(np.float32).reshape(X.shape + (-1,)) out = np.concatenate((X0,Y,R), axis=1) np.savetxt(f, out, delimiter=',') diff --git a/Experiment.py b/Experiment.py index 33cc9dc..c6c04f6 100644 --- a/Experiment.py +++ b/Experiment.py @@ -58,8 +58,8 @@ # Now Lets Simplify the structure (Note this requires everything to be a float) # If you get an error that the input size isn't right, try changing float below to float32 or float64 -Train_X = Train_X.view(float).reshape(Train_X.shape + (-1,)) -Test_X = Test_X.view(float).reshape(Test_X.shape + (-1,)) +Train_X = Train_X.view(np.float32).reshape(Train_X.shape + (-1,)) +Test_X = Test_X.view(np.float32).reshape(Test_X.shape + (-1,)) # Protect against divide by zero! Train_X = np.nan_to_num(Train_X) @@ -70,6 +70,7 @@ Width = sum ( [ len(FieldGroups[i]) for i in xrange(len(FieldGroups)) ] ) N_Classes = np.shape(Train_Y)[1] print 'N Inputs:', N_Inputs +print 'Width:', Width print 'N Classes:', N_Classes # Now Build the Model diff --git a/SUSYConfig.py b/SUSYConfig.py index 1bddba1..fee30ec 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -150,10 +150,8 @@ 'Nesterov': 0., 'WeightInitialization':"'normal'"} -#Params = {'Width': [128], -# 'Depth': [2], -# 'loss': ["'categorical_crossentropy'"]} -Params = {'Depth': [2]} +Params = {'Depth': [2], + 'loss': ["'categorical_crossentropy'"]} PS = Permutator (Params) Combos = PS.Permutations () diff --git a/ZllConfig.py b/ZllConfig.py index 41cb23a..f763c9e 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -49,10 +49,8 @@ 'Nesterov': 0., 'WeightInitialization':"'normal'"} -#Params = {'Width': [8], -# 'Depth': [2], -# 'loss': ["'categorical_crossentropy'"]} -Params = {'Depth': [2]} +Params = {'Depth': [2], + 'loss': ["'categorical_crossentropy'"]} PS = Permutator (Params) Combos = PS.Permutations () From 5160fd794fc6a6fae14316ab0c0fc1a3ea655e21 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 10 Nov 2016 22:26:00 -0600 Subject: [PATCH 27/38] tested lxplus, checking uta --- CSVWriter.py | 1 + Experiment.py | 2 ++ ZllConfig.py | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CSVWriter.py b/CSVWriter.py index 31733a2..45a426a 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -14,6 +14,7 @@ def CSVWriter (filename, X, Y, R): f.write(','.join(colnames) + '\n') X0 = X.view(np.float32).reshape(X.shape + (-1,)) + #X0 = X.view(float).reshape(X.shape + (-1,)) out = np.concatenate((X0,Y,R), axis=1) np.savetxt(f, out, delimiter=',') diff --git a/Experiment.py b/Experiment.py index c6c04f6..9feadaf 100644 --- a/Experiment.py +++ b/Experiment.py @@ -60,6 +60,8 @@ # If you get an error that the input size isn't right, try changing float below to float32 or float64 Train_X = Train_X.view(np.float32).reshape(Train_X.shape + (-1,)) Test_X = Test_X.view(np.float32).reshape(Test_X.shape + (-1,)) +#Train_X = Train_X.view(float).reshape(Train_X.shape + (-1,)) +#Test_X = Test_X.view(float).reshape(Test_X.shape + (-1,)) # Protect against divide by zero! Train_X = np.nan_to_num(Train_X) diff --git a/ZllConfig.py b/ZllConfig.py index f763c9e..b9c61ca 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -41,7 +41,7 @@ Name = 'ZllModel' Config = {'MaxEvents': 50000, - 'Epochs': 10000, + 'Epochs': 1000, 'BatchSize': 2048*8, 'LearningRate': 0.005, 'Decay': 0., From 25a80473f052e0e5304e83c91917da05793aad55 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 10 Nov 2016 22:27:47 -0600 Subject: [PATCH 28/38] tested lxplus, checking uta --- SUSYConfig.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/SUSYConfig.py b/SUSYConfig.py index fee30ec..a24a5c7 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -55,17 +55,17 @@ ['L1_phi', 'L2_phi','B1_phi', 'B2_phi'], # decay angles (by frame) - ['cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB'] - ['cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB'] - ['cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB'] - ['cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB'] - ['cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB'] + ['cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB'], + ['cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB'], + ['cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB'], + ['cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB'], + ['cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB'], # separation angles (by frame) - ['dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA'] - ['dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA'] - ['dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA'] - ['dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA'] + ['dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA'], + ['dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA'], + ['dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA'], + ['dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA'], ] SelectedFields = [ From 8599efcb429f3578d221dc910a7608e4c1ad2a13 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Fri, 11 Nov 2016 10:11:07 -0600 Subject: [PATCH 29/38] actually quick --- ZllConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZllConfig.py b/ZllConfig.py index b9c61ca..f763c9e 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -41,7 +41,7 @@ Name = 'ZllModel' Config = {'MaxEvents': 50000, - 'Epochs': 1000, + 'Epochs': 10000, 'BatchSize': 2048*8, 'LearningRate': 0.005, 'Decay': 0., From c1e81d2cfb93001ac4c52699d8276be5b780312a Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Fri, 11 Nov 2016 13:51:10 -0600 Subject: [PATCH 30/38] optimizing --- Experiment.py | 2 +- SUSYConfig.py | 15 ++++++++------- ZllConfig.py | 6 ++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Experiment.py b/Experiment.py index 9feadaf..f041348 100644 --- a/Experiment.py +++ b/Experiment.py @@ -69,7 +69,7 @@ # Get some Inof N_Inputs = len(SelectedFields[VarSet]) -Width = sum ( [ len(FieldGroups[i]) for i in xrange(len(FieldGroups)) ] ) +#Width = sum ( [ len(FieldGroups[i]) for i in xrange(len(FieldGroups)) ] ) N_Classes = np.shape(Train_Y)[1] print 'N Inputs:', N_Inputs print 'Width:', Width diff --git a/SUSYConfig.py b/SUSYConfig.py index a24a5c7..b888feb 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -71,11 +71,11 @@ SelectedFields = [ # all observables ['mP', 'mC', 'mX', + 'METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', - 'METx', 'METy', 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', @@ -90,19 +90,19 @@ 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], # detector reconstruction - ['L1_pT', 'L1_eta', 'L1_phi', 'L1_M', + ['METx', 'METy', + 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', - 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', - 'METx', 'METy'], + 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M'], # physics reconstruction ['mP', 'mC', 'mX', + 'METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', - 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', - 'METx', 'METy'], + 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M'], # recursive jigsaw reconstruction ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', @@ -150,7 +150,8 @@ 'Nesterov': 0., 'WeightInitialization':"'normal'"} -Params = {'Depth': [2], +Params = {'Width': [1024], + 'Depth': [2], 'loss': ["'categorical_crossentropy'"]} PS = Permutator (Params) diff --git a/ZllConfig.py b/ZllConfig.py index f763c9e..8fda932 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -34,8 +34,9 @@ ] SelectedFields = [ + # all variables ['LP_pT', 'LP_eta', 'LP_phi', 'LP_E', - 'LM_pT', 'LM_eta', 'LM_phi', 'LM_E'] + 'LM_pT', 'LM_eta', 'LM_phi', 'LM_E'], ] Name = 'ZllModel' @@ -49,7 +50,8 @@ 'Nesterov': 0., 'WeightInitialization':"'normal'"} -Params = {'Depth': [2], +Params = {'Width': [1024], + 'Depth': [2], 'loss': ["'categorical_crossentropy'"]} PS = Permutator (Params) From 0738ed5fd2909fccb91180249277b2722a41fc32 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 1 Dec 2016 14:41:44 -0600 Subject: [PATCH 31/38] testing new normalization --- Arguments.py | 2 ++ CSVWriter.py | 5 ++--- Experiment.py | 43 +++++++++++------------------------ SUSYConfig.py | 62 +++++++++++++++++++++++---------------------------- ZllConfig.py | 24 ++++++++------------ 5 files changed, 54 insertions(+), 82 deletions(-) diff --git a/Arguments.py b/Arguments.py index 377dd06..11baa02 100644 --- a/Arguments.py +++ b/Arguments.py @@ -1,3 +1,5 @@ +import argparse + # Configuration of this job parser = argparse.ArgumentParser() parser.add_argument ('-C', '--config', default='EventClassificationDNN/ScanConfig.py') diff --git a/CSVWriter.py b/CSVWriter.py index 45a426a..8dd4b5d 100644 --- a/CSVWriter.py +++ b/CSVWriter.py @@ -2,7 +2,7 @@ import numpy as np -def CSVWriter (filename, X, Y, R): +def CSVWriter (filename, X, Y, R, arrType): names = X.dtype.names colnames = [] @@ -13,8 +13,7 @@ def CSVWriter (filename, X, Y, R): f = open(filename, 'w') f.write(','.join(colnames) + '\n') - X0 = X.view(np.float32).reshape(X.shape + (-1,)) - #X0 = X.view(float).reshape(X.shape + (-1,)) + X0 = X.view(arrType).reshape(X.shape + (-1,)) out = np.concatenate((X0,Y,R), axis=1) np.savetxt(f, out, delimiter=',') diff --git a/Experiment.py b/Experiment.py index f041348..0049113 100644 --- a/Experiment.py +++ b/Experiment.py @@ -1,12 +1,10 @@ -import sys, os, argparse - -import h5py -import numpy as np +import os # Parse Arguments execfile ('EventClassificationDNN/Arguments.py') # Now load the Hyperparameters +import numpy as np execfile (ConfigFile) if 'Config' in dir(): @@ -14,38 +12,26 @@ pass # Load the Data +import h5py from EventClassificationDNN.MultiClassTools import * -#from EventClassificationDNN.InputFiles import Samples (Train_X, Train_Y), (Test_X, Test_Y), ClassIndex = LoadData (Samples, .1, MaxEvents=MaxEvents) # Select Variables To use in training # To get the field names, just look at Fields=Train_X.dtype.names -#from EventClassificationDNN.InputVars import FieldGroups, SelectedFields # Keep the original data before renomalizing... will use this in output Train_X0 = Train_X.copy() Test_X0 = Test_X.copy() -GroupMins = [0] * len(FieldGroups) -GroupMaxs = [0] * len(FieldGroups) - # Normalize Ranges within variable groups e.g. masses, angles (phi, eta, cos separately) -for Fs in xrange(len(FieldGroups)): - Mins = [] - Maxs = [] - for varI in xrange(len(FieldGroups[Fs])): - Mins.append ( np.min(Train_X0[FieldGroups[Fs][varI]]) ) - Maxs.append ( np.max(Train_X0[FieldGroups[Fs][varI]]) ) - pass - GroupMins[Fs] = min(Mins) - GroupMaxs[Fs] = max(Maxs) - for var in FieldGroups[Fs]: - yy = Train_X[var] - yy[:] = 1./(GroupMaxs[Fs]-GroupMins[Fs]) * (yy-GroupMins[Fs]) - yy1 = Test_X[var] - yy1[:] = 1./(GroupMaxs[Fs]-GroupMins[Fs])* (yy1-GroupMins[Fs]) - pass +for Fs in xrange(len(Observables)): + M = np.mean(Train_X0[Observables[Fs]]) + V = np.var(Train_X0[Observables[Fs]]) + yy = Train_X[Observables[Fs]] + yy[:] = (yy - M) / V + .5 + yy1 = Train_X[Observables[Fs]] + yy1[:] = (yy1 - M) / V + .5 pass #Train_X_N = Train_X @@ -58,10 +44,8 @@ # Now Lets Simplify the structure (Note this requires everything to be a float) # If you get an error that the input size isn't right, try changing float below to float32 or float64 -Train_X = Train_X.view(np.float32).reshape(Train_X.shape + (-1,)) -Test_X = Test_X.view(np.float32).reshape(Test_X.shape + (-1,)) -#Train_X = Train_X.view(float).reshape(Train_X.shape + (-1,)) -#Test_X = Test_X.view(float).reshape(Test_X.shape + (-1,)) +Train_X = Train_X.view(arrType).reshape(Train_X.shape + (-1,)) +Test_X = Test_X.view(arrType).reshape(Test_X.shape + (-1,)) # Protect against divide by zero! Train_X = np.nan_to_num(Train_X) @@ -69,7 +53,6 @@ # Get some Inof N_Inputs = len(SelectedFields[VarSet]) -#Width = sum ( [ len(FieldGroups[i]) for i in xrange(len(FieldGroups)) ] ) N_Classes = np.shape(Train_Y)[1] print 'N Inputs:', N_Inputs print 'Width:', Width @@ -127,5 +110,5 @@ if WriteResults: print 'Writing Results.' from EventClassificationDNN.CSVWriter import * - CSVWriter (MyModel.OutDir+'/Result.csv', Test_X0, Test_Y, result) + CSVWriter (MyModel.OutDir+'/Result.csv', Test_X0, Test_Y, result, arrType) pass diff --git a/SUSYConfig.py b/SUSYConfig.py index b888feb..f33e031 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -1,7 +1,4 @@ -import random -import getopt from DLTools.Permutator import * -import sys,argparse #Previously in InputFiles.py # Define the Input Files @@ -18,54 +15,50 @@ 'mP1000_mC800_mX100.h5'] # Select datasets (formerly TTrees in original ROOT file) -Files = [] +Samples = [] for InputData in InputFiles: InputData = os.getenv('SampleDir') + '/' + InputData - Files += [ [InputData, 'AA_Gen'], - [InputData, 'AB_Gen'], - [InputData, 'BA_Gen'], - [InputData, 'BB_Gen'] ] + Sanples += [ [InputData, 'AA_Gen'], + [InputData, 'AB_Gen'], + [InputData, 'BA_Gen'], + [InputData, 'BB_Gen'] ] pass -Samples = [] - -for F in Files: Samples.append(F) - #Previously in InputVars.py # Select Variables To use in training # used for scaling -FieldGroups = [ +Observables = [ # energy scale - ['mP', 'mC', 'mX', - 'METx', 'METy', - 'L1_pT', 'L1_M', - 'L2_pT','L2_M', - 'B1_pT','B1_M', - 'B2_pT','B2_M', - 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', - 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], + 'mP', 'mC', 'mX', + 'METx', 'METy', + 'L1_pT', 'L1_M', + 'L2_pT','L2_M', + 'B1_pT','B1_M', + 'B2_pT','B2_M', + 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', + 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', # pseudo rapidity - ['L1_eta', 'L2_eta', 'B1_eta','B2_eta'], + 'L1_eta', 'L2_eta', 'B1_eta','B2_eta', # azimutahl angle - ['L1_phi', 'L2_phi','B1_phi', 'B2_phi'], + 'L1_phi', 'L2_phi','B1_phi', 'B2_phi', # decay angles (by frame) - ['cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB'], - ['cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB'], - ['cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB'], - ['cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB'], - ['cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB'], + 'cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB', + 'cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB', + 'cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB', + 'cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB', + 'cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB', # separation angles (by frame) - ['dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA'], - ['dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA'], - ['dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA'], - ['dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA'], + 'dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA', + 'dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA', + 'dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA', + 'dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA', ] SelectedFields = [ @@ -148,6 +141,7 @@ 'Decay': 0., 'Momentum': 0., 'Nesterov': 0., + 'arrType': float, 'WeightInitialization':"'normal'"} Params = {'Width': [1024], diff --git a/ZllConfig.py b/ZllConfig.py index 8fda932..aadd83e 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -1,36 +1,29 @@ -import random -import getopt from DLTools.Permutator import * -import sys,argparse #Previously in InputFiles.py # Define the Input Files InputFiles = ['Zll.h5'] # Select datasets (formerly TTrees in original ROOT file) -Files = [] +Samples = [] for InputData in InputFiles: InputData = os.getenv('SampleDir') + '/' + InputData - Files += [ [InputData, 'Zto2LOS'], - [InputData, 'Rndm2LOS'] ] + Samples += [ [InputData, 'Zto2LOS'], + [InputData, 'Rndm2LOS'] ] pass -Samples = [] - -for F in Files: Samples.append(F) - #Previously in InputVars.py # Select Variables To use in training -FieldGroups = [ +Observables = [ # energy scale - ['LP_pT', 'LP_E', - 'LM_pT', 'LM_E'], + 'LP_pT', 'LP_E', + 'LM_pT', 'LM_E', # transverse angle - ['LP_phi', 'LM_phi'], + 'LP_phi', 'LM_phi', # pseudorapidity - ['LP_eta', 'LM_eta'], + 'LP_eta', 'LM_eta', ] SelectedFields = [ @@ -48,6 +41,7 @@ 'Decay': 0., 'Momentum': 0., 'Nesterov': 0., + 'arrType': np.float32, 'WeightInitialization':"'normal'"} Params = {'Width': [1024], From d5e4ec95f123fad9c4b147f3df84b5b9744b409a Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 1 Dec 2016 22:09:51 -0600 Subject: [PATCH 32/38] IQR weight --- Experiment.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Experiment.py b/Experiment.py index 0049113..f6faab0 100644 --- a/Experiment.py +++ b/Experiment.py @@ -25,13 +25,25 @@ Test_X0 = Test_X.copy() # Normalize Ranges within variable groups e.g. masses, angles (phi, eta, cos separately) -for Fs in xrange(len(Observables)): - M = np.mean(Train_X0[Observables[Fs]]) - V = np.var(Train_X0[Observables[Fs]]) - yy = Train_X[Observables[Fs]] - yy[:] = (yy - M) / V + .5 - yy1 = Train_X[Observables[Fs]] - yy1[:] = (yy1 - M) / V + .5 +#for Fs in xrange(len(Observables)): +# M = np.mean(Train_X0[Observables[Fs]]) +# V = np.var(Train_X0[Observables[Fs]]) +# yy = Train_X[Observables[Fs]] +# yy[:] = (yy - M) / V + .5 +# yy1 = Train_X[Observables[Fs]] +# yy1[:] = (yy1 - M) / V + .5 +# pass +# IQR +for obs in xrange(len(Observables)): + nanpct = numpy.nanpercentile (Train_X0[Observables[obs]], numpy.arange(2500,7500)/100.) + yy = Train_X[Observables[obs]] + yy1 = Train_x[Observables[obs]] + for itrv in xrange(len(yy)): + for itrw in xrange(len(nanpct)): + if nanpct[itrw] >= yy[itrv]: yy[itrv] = itrw/10000. + break + yy[itrv] = itrw/10000. + pass pass #Train_X_N = Train_X From 49973c609862c6d8888df7241fff1b0365ce7846 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 1 Dec 2016 22:48:38 -0600 Subject: [PATCH 33/38] scale/shift --- Experiment.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/Experiment.py b/Experiment.py index f6faab0..bf8e591 100644 --- a/Experiment.py +++ b/Experiment.py @@ -24,7 +24,7 @@ Train_X0 = Train_X.copy() Test_X0 = Test_X.copy() -# Normalize Ranges within variable groups e.g. masses, angles (phi, eta, cos separately) +# Normalize Ranges #for Fs in xrange(len(Observables)): # M = np.mean(Train_X0[Observables[Fs]]) # V = np.var(Train_X0[Observables[Fs]]) @@ -33,16 +33,22 @@ # yy1 = Train_X[Observables[Fs]] # yy1[:] = (yy1 - M) / V + .5 # pass -# IQR +#### IQR shifted to [0,1] (set outliers to -1) for obs in xrange(len(Observables)): - nanpct = numpy.nanpercentile (Train_X0[Observables[obs]], numpy.arange(2500,7500)/100.) + print 'Normalizing', Observables[obs], '...' + nanpct = numpy.nanpercentile (Train_X0[Observables[obs]], numpy.arange(2500,7501)/100.) yy = Train_X[Observables[obs]] yy1 = Train_x[Observables[obs]] for itrv in xrange(len(yy)): + val = yy[itrv] for itrw in xrange(len(nanpct)): - if nanpct[itrw] >= yy[itrv]: yy[itrv] = itrw/10000. + if nanpct[itrw] >= val: + weight = itrw/10000. + shift = 2. * weight - .5 + yy[itrv] = itrw/10000. + pass break - yy[itrv] = itrw/10000. + yy[itrv] = -1. pass pass From c08a0eedbff776ca6e33cfba97131834e7f97749 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Thu, 1 Dec 2016 22:54:54 -0600 Subject: [PATCH 34/38] capitalization --- Experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Experiment.py b/Experiment.py index bf8e591..9544878 100644 --- a/Experiment.py +++ b/Experiment.py @@ -38,7 +38,7 @@ print 'Normalizing', Observables[obs], '...' nanpct = numpy.nanpercentile (Train_X0[Observables[obs]], numpy.arange(2500,7501)/100.) yy = Train_X[Observables[obs]] - yy1 = Train_x[Observables[obs]] + yy1 = Train_X[Observables[obs]] for itrv in xrange(len(yy)): val = yy[itrv] for itrw in xrange(len(nanpct)): From 7c0e1598b1a4d30bb51a6141cdca71997f2872d9 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Wed, 14 Dec 2016 14:30:57 -0600 Subject: [PATCH 35/38] set for tutorial again --- Experiment.py | 101 ++++++++++++++++++++++++++++++++++++++------------ SUSYConfig.py | 58 +++++++++++++---------------- ZllConfig.py | 32 ++++++++-------- 3 files changed, 120 insertions(+), 71 deletions(-) diff --git a/Experiment.py b/Experiment.py index 9544878..114976f 100644 --- a/Experiment.py +++ b/Experiment.py @@ -1,3 +1,7 @@ +import datetime +a = datetime.datetime.now() +print a.ctime() + import os # Parse Arguments @@ -24,31 +28,77 @@ Train_X0 = Train_X.copy() Test_X0 = Test_X.copy() -# Normalize Ranges -#for Fs in xrange(len(Observables)): -# M = np.mean(Train_X0[Observables[Fs]]) -# V = np.var(Train_X0[Observables[Fs]]) -# yy = Train_X[Observables[Fs]] -# yy[:] = (yy - M) / V + .5 -# yy1 = Train_X[Observables[Fs]] -# yy1[:] = (yy1 - M) / V + .5 -# pass -#### IQR shifted to [0,1] (set outliers to -1) -for obs in xrange(len(Observables)): - print 'Normalizing', Observables[obs], '...' - nanpct = numpy.nanpercentile (Train_X0[Observables[obs]], numpy.arange(2500,7501)/100.) - yy = Train_X[Observables[obs]] - yy1 = Train_X[Observables[obs]] - for itrv in xrange(len(yy)): - val = yy[itrv] - for itrw in xrange(len(nanpct)): - if nanpct[itrw] >= val: - weight = itrw/10000. - shift = 2. * weight - .5 - yy[itrv] = itrw/10000. +#### Normalization +for scheme in Observables: + if scheme == 'Group': + for grp in Observables[scheme]: + Mins = [] + Maxs = [] + for obs in grp: + Mins.append ( np.min(Train_X0[obs]) ) + Maxs.append ( np.max(Train_X0[obs]) ) + pass + minval = min(Mins) + maxval = max(Maxs) + for obs in grp: + yy = Train_X[obs] + yy[:] = 1./(maxval-minval) * (yy-minval) + yy1 = Test_X[obs] + yy1[:] = 1./(maxval-minval) * (yy1-minval) + pass + pass + continue + for obs in Observables[scheme]: + print 'Normalizing', obs, 'to', scheme, '...' + yy = Train_X[obs] + yy1 = Train_X[obs] + if scheme == 'Gaus': # Gaus variance set to [0,1] + M = np.mean (Train_X0[obs]) + V = np.var (Train_X0[obs]) + yy[:] = (yy - M) / V + .5 + yy1[:] = (yy1 - M) / V + .5 + pass + else: + if scheme == 'Full': + lower = 0 + upper = 10001 + pass + elif scheme == 'LeftTrim': + lower = 2500 + upper = 10001 pass - break - yy[itrv] = -1. + elif scheme == 'RightTrim': + lower = 0 + upper = 7501 + pass + elif scheme == 'IQR': + lower = 2500 + upper = 7501 + pass + nanpct = np.nanpercentile (Train_X0[obs], np.arange(lower,upper)/100.) + for itrv in xrange(len(yy)): + val = yy[itrv] + if np.isnan(val) or np.isinf(val) or np.isneginf(val): continue + for itrw in xrange(len(nanpct)): + weight = (lower+itrw)/10000. + if val >= weight: + # left-trimmed range shifted to [0, 1] + if scheme == 'LeftTrim': shift = 4. * (weight-.25) / 3. + # right-trimmed range shifted to [0, 1] + elif scheme == 'RightTrim': shift = 4. * weight / 3. + # IQR shifted to [0, 1] + elif scheme == 'IQR': shift = 2. * weight - .5 + # full range shifted to [0,1] + else: shift = weight + yy[itrv] = shift + yy1[itrv] = shift + pass + break + # default to NaN + yy[itrv] = np.nan + yy1[itrv] = np.nan + pass + pass pass pass @@ -130,3 +180,6 @@ from EventClassificationDNN.CSVWriter import * CSVWriter (MyModel.OutDir+'/Result.csv', Test_X0, Test_Y, result, arrType) pass + +a = datetime.datetime.now() +print a.ctime() diff --git a/SUSYConfig.py b/SUSYConfig.py index f33e031..c094a20 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -28,38 +28,32 @@ # Select Variables To use in training # used for scaling -Observables = [ - # energy scale - 'mP', 'mC', 'mX', - 'METx', 'METy', - 'L1_pT', 'L1_M', - 'L2_pT','L2_M', - 'B1_pT','B1_M', - 'B2_pT','B2_M', - 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', - 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', - - # pseudo rapidity - 'L1_eta', 'L2_eta', 'B1_eta','B2_eta', - - # azimutahl angle - 'L1_phi', 'L2_phi','B1_phi', 'B2_phi', - - # decay angles (by frame) - 'cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB', - 'cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB', - 'cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB', - 'cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB', - 'cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB', - - # separation angles (by frame) - 'dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA', - 'dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA', - 'dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA', - 'dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA', -] +Observables = { + # keep the full range + 'Full': ['mP', 'mC', 'mX', + 'L1_eta', 'L2_eta', 'B1_eta', 'B2_eta', + 'L1_phi', 'L2_phi','B1_phi', 'B2_phi', + 'cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB', + 'cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB', + 'cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB', + 'cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB', + 'cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB', + 'dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA', + 'dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA', + 'dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA', + 'dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA'], + + # keep IQR + 'IQR': ['METx', 'METy', + 'L1_pT', 'L1_M', + 'L2_pT', 'L2_M', + 'B1_pT', 'B1_M', + 'B2_pT', 'B2_M', + 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', + 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', + 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], +} SelectedFields = [ # all observables diff --git a/ZllConfig.py b/ZllConfig.py index aadd83e..e907131 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -14,40 +14,42 @@ #Previously in InputVars.py # Select Variables To use in training -Observables = [ - # energy scale - 'LP_pT', 'LP_E', - 'LM_pT', 'LM_E', - - # transverse angle - 'LP_phi', 'LM_phi', - - # pseudorapidity - 'LP_eta', 'LM_eta', -] +Observables = { + # group similar + 'Group': [ + ['LP_pT', 'LP_E', + 'LM_pT', 'LM_E'], + ['LP_phi', 'LM_phi'], + ['LP_eta', 'LM_eta'], + ], +} SelectedFields = [ # all variables ['LP_pT', 'LP_eta', 'LP_phi', 'LP_E', 'LM_pT', 'LM_eta', 'LM_phi', 'LM_E'], + + # known differences + ['LP_phi', 'LM_phi'], ] Name = 'ZllModel' Config = {'MaxEvents': 50000, - 'Epochs': 10000, + 'Epochs': 1000, 'BatchSize': 2048*8, 'LearningRate': 0.005, 'Decay': 0., 'Momentum': 0., 'Nesterov': 0., - 'arrType': np.float32, 'WeightInitialization':"'normal'"} -Params = {'Width': [1024], - 'Depth': [2], +Params = {'Width': [1585], + 'Depth': [1], 'loss': ["'categorical_crossentropy'"]} +arrType = np.float32 + PS = Permutator (Params) Combos = PS.Permutations () From ee483c0820c725ba138f1796b308910c83553846 Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 10 Jan 2017 14:11:21 -0600 Subject: [PATCH 36/38] retest --- Experiment.py | 127 ++++++++++++++++++++++---------------------------- SUSYConfig.py | 126 ++++++++++++++++++++++++++++++++----------------- ZllConfig.py | 20 ++++---- 3 files changed, 149 insertions(+), 124 deletions(-) diff --git a/Experiment.py b/Experiment.py index 114976f..aeb92e0 100644 --- a/Experiment.py +++ b/Experiment.py @@ -12,7 +12,9 @@ execfile (ConfigFile) if 'Config' in dir(): - for a in Config: exec (a + '=' + str(Config[a])) + for a in Config: + if a != 'arrType': exec (a + '=' + str(Config[a])) + pass pass # Load the Data @@ -28,80 +30,63 @@ Train_X0 = Train_X.copy() Test_X0 = Test_X.copy() -#### Normalization -for scheme in Observables: - if scheme == 'Group': - for grp in Observables[scheme]: - Mins = [] - Maxs = [] - for obs in grp: - Mins.append ( np.min(Train_X0[obs]) ) - Maxs.append ( np.max(Train_X0[obs]) ) - pass - minval = min(Mins) - maxval = max(Maxs) - for obs in grp: - yy = Train_X[obs] - yy[:] = 1./(maxval-minval) * (yy-minval) - yy1 = Test_X[obs] - yy1[:] = 1./(maxval-minval) * (yy1-minval) - pass +#### Trimming +print 'Trimming outliers from data ...' +num = Train_X.shape[0] +for obs in Observables: + if not Observables[obs].has_key('trim'): continue + print ' ', obs + if Observables[obs]['trim'][0] < 0: lower = -np.inf + else: lower = np.nanpercentile(Train_X[obs], Observables[obs]['trim'][0]) + if Observables[obs]['trim'][1] > 100: upper = np.inf + else: upper = np.nanpercentile(Train_X[obs], Observables[obs]['trim'][1]) + for evt in xrange(num): + val = Train_X[obs][evt] + if np.isnan(val) or np.isinf(val) or np.isneginf(val): + Train_X[obs][evt] = np.nan + pass + elif val > upper: + Train_X[obs][evt] = np.nan pass - continue - for obs in Observables[scheme]: - print 'Normalizing', obs, 'to', scheme, '...' - yy = Train_X[obs] - yy1 = Train_X[obs] - if scheme == 'Gaus': # Gaus variance set to [0,1] - M = np.mean (Train_X0[obs]) - V = np.var (Train_X0[obs]) - yy[:] = (yy - M) / V + .5 - yy1[:] = (yy1 - M) / V + .5 + elif val < lower: + Train_X[obs][val] = np.nan pass - else: - if scheme == 'Full': - lower = 0 - upper = 10001 - pass - elif scheme == 'LeftTrim': - lower = 2500 - upper = 10001 - pass - elif scheme == 'RightTrim': - lower = 0 - upper = 7501 - pass - elif scheme == 'IQR': - lower = 2500 - upper = 7501 - pass - nanpct = np.nanpercentile (Train_X0[obs], np.arange(lower,upper)/100.) - for itrv in xrange(len(yy)): - val = yy[itrv] - if np.isnan(val) or np.isinf(val) or np.isneginf(val): continue - for itrw in xrange(len(nanpct)): - weight = (lower+itrw)/10000. - if val >= weight: - # left-trimmed range shifted to [0, 1] - if scheme == 'LeftTrim': shift = 4. * (weight-.25) / 3. - # right-trimmed range shifted to [0, 1] - elif scheme == 'RightTrim': shift = 4. * weight / 3. - # IQR shifted to [0, 1] - elif scheme == 'IQR': shift = 2. * weight - .5 - # full range shifted to [0,1] - else: shift = weight - yy[itrv] = shift - yy1[itrv] = shift - pass - break - # default to NaN - yy[itrv] = np.nan - yy1[itrv] = np.nan - pass + pass + pass +for evt in xrange(num): + if any(np.isnan(Train_X[obs][evt]) for obs in Observables): + for obs in Observables: + Train_X[obs][evt] = np.nan pass pass pass +#### Shifting/Scaling normalization +print 'Normalizing', obs, '...' +for obs in Observables: + print ' ', obs + if Observables[obs].has_key('range'): + minval = Observables[obs]['range'][0] + maxval = Observables[obs]['range'][1] + pass + else: + minval = np.min(Train_X[obs]) + maxval = np.max(Train_X[obs]) + pass + yy = Train_X[obs] + yy[:] = 1./(maxval-minval) * (yy-minval) + yy1 = Test_X[obs] + yy1[:] = 1./(maxval-minval) * (yy1-minval) + # gaus + yy = Train_X[obs] + yy1 = Train_X[obs] + M = np.mean (Train_X0[obs]) + V = np.var (Train_X0[obs]) + yy[:] = (yy - M) / V + .5 + yy1[:] = (yy1 - M) / V + .5 + pass +pass + #Train_X_N = Train_X # Keep Only selected Variables @@ -112,8 +97,8 @@ # Now Lets Simplify the structure (Note this requires everything to be a float) # If you get an error that the input size isn't right, try changing float below to float32 or float64 -Train_X = Train_X.view(arrType).reshape(Train_X.shape + (-1,)) -Test_X = Test_X.view(arrType).reshape(Test_X.shape + (-1,)) +Train_X = Train_X.view(Config['arrType']).reshape(Train_X.shape + (-1,)) +Test_X = Test_X.view(Config['arrType']).reshape(Test_X.shape + (-1,)) # Protect against divide by zero! Train_X = np.nan_to_num(Train_X) diff --git a/SUSYConfig.py b/SUSYConfig.py index c094a20..476212e 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -17,8 +17,8 @@ # Select datasets (formerly TTrees in original ROOT file) Samples = [] for InputData in InputFiles: - InputData = os.getenv('SampleDir') + '/' + InputData - Sanples += [ [InputData, 'AA_Gen'], + InputData = os.getenv('SampleDirSUSY') + '/' + InputData + Samples += [ [InputData, 'AA_Gen'], [InputData, 'AB_Gen'], [InputData, 'BA_Gen'], [InputData, 'BB_Gen'] ] @@ -29,31 +29,84 @@ # used for scaling Observables = { - # keep the full range - 'Full': ['mP', 'mC', 'mX', - 'L1_eta', 'L2_eta', 'B1_eta', 'B2_eta', - 'L1_phi', 'L2_phi','B1_phi', 'B2_phi', - 'cosPP_AA', 'cosPP_AB', 'cosPP_BA', 'cosPP_BB', - 'cosPa_AA', 'cosPa_AB', 'cosPa_BA', 'cosPa_BB', - 'cosPb_AA', 'cosPb_AB', 'cosPb_BA', 'cosPb_BB', - 'cosCa_AA', 'cosCa_AB', 'cosCa_BA', 'cosCa_BB', - 'cosCb_AA', 'cosCb_AB', 'cosCb_BA', 'cosCb_BB', - 'dphi_PP_Pa_AA', 'dphi_PP_Pa_BB', 'dphi_PP_Pa_AB', 'dphi_PP_Pa_BA', - 'dphi_PP_Pb_AA', 'dphi_PP_Pb_BB', 'dphi_PP_Pb_AB', 'dphi_PP_Pb_BA', - 'dphi_Pa_Ca_AA', 'dphi_Pa_Ca_BB', 'dphi_Pa_Ca_AB', 'dphi_Pa_Ca_BA', - 'dphi_Pb_Cb_AA', 'dphi_Pb_Cb_BB', 'dphi_Pb_Cb_AB', 'dphi_Pb_Cb_BA'], - - # keep IQR - 'IQR': ['METx', 'METy', - 'L1_pT', 'L1_M', - 'L2_pT', 'L2_M', - 'B1_pT', 'B1_M', - 'B2_pT', 'B2_M', - 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', - 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA'], -} + 'mP': {}, + 'mC': {}, + 'mX': {}, + 'B1_pT': {'trim': [0, 99]}, + 'B1_phi': {'range': [-np.pi, np.pi]}, + 'B1_eta': {'range': [-2.5, 2.5]}, + 'B1_M': {'trim': [0, 99]}, + 'B2_pT': {'trim': [0, 99]}, + 'B2_phi': {'range': [-np.pi, np.pi]}, + 'B2_eta': {'range': [-2.5, 2.5]}, + 'B2_M': {'trim': [0, 99]}, + 'L1_pT': {'trim': [0, 99]}, + 'L1_phi': {'range': [-np.pi, np.pi]}, + 'L1_eta': {'range': [-2.5, 2.5]}, + 'L1_M': {'trim': [0, 99]}, + 'L2_pT': {'trim': [0, 99]}, + 'L2_phi': {'range': [-np.pi, np.pi]}, + 'L2_eta': {'range': [-2.5, 2.5]}, + 'L2_M': {'trim': [0, 99]}, + 'METx': {'trim': [0, 99]}, + 'METy': {'trim': [0, 99]}, + 'MPP_AA': {}, + 'Eb_a_AA': {}, + 'Eb_b_AA': {}, + 'El_a_AA': {}, + 'El_b_AA': {}, + 'MPP_BB': {}, + 'Eb_a_BB': {}, + 'Eb_b_BB': {}, + 'El_a_BB': {}, + 'El_b_BB': {}, + 'MPP_AB': {}, + 'Eb_a_AB': {}, + 'Eb_b_AB': {}, + 'El_a_AB': {}, + 'El_b_AB': {}, + 'MPP_BA': {}, + 'Eb_a_BA': {}, + 'Eb_b_BA': {}, + 'El_a_BA': {}, + 'El_b_BA': {}, + 'cosPP_AA': {'range': [0, 1]}, + 'cosPP_AB': {'range': [0, 1]}, + 'cosPP_BA': {'range': [0, 1]}, + 'cosPP_BB': {'range': [0, 1]}, + 'cosPa_AA': {'range': [0, 1]}, + 'cosPa_AB': {'range': [0, 1]}, + 'cosPa_BA': {'range': [0, 1]}, + 'cosPa_BB': {'range': [0, 1]}, + 'cosPb_AA': {'range': [0, 1]}, + 'cosPb_AB': {'range': [0, 1]}, + 'cosPb_BA': {'range': [0, 1]}, + 'cosPb_BB': {'range': [0, 1]}, + 'cosCa_AA': {'range': [0, 1]}, + 'cosCa_AB': {'range': [0, 1]}, + 'cosCa_BA': {'range': [0, 1]}, + 'cosCa_BB': {'range': [0, 1]}, + 'cosCb_AA': {'range': [0, 1]}, + 'cosCb_AB': {'range': [0, 1]}, + 'cosCb_BA': {'range': [0, 1]}, + 'cosCb_BB': {'range': [0, 1]}, + 'dphi_PP_Pa_AA': {'range': [0, np.pi]}, + 'dphi_PP_Pa_BB': {'range': [0, np.pi]}, + 'dphi_PP_Pa_AB': {'range': [0, np.pi]}, + 'dphi_PP_Pa_BA': {'range': [0, np.pi]}, + 'dphi_PP_Pb_AA': {'range': [0, np.pi]}, + 'dphi_PP_Pb_BB': {'range': [0, np.pi]}, + 'dphi_PP_Pb_AB': {'range': [0, np.pi]}, + 'dphi_PP_Pb_BA': {'range': [0, np.pi]}, + 'dphi_Pa_Ca_AA': {'range': [0, np.pi]}, + 'dphi_Pa_Ca_BB': {'range': [0, np.pi]}, + 'dphi_Pa_Ca_AB': {'range': [0, np.pi]}, + 'dphi_Pa_Ca_BA': {'range': [0, np.pi]}, + 'dphi_Pb_Cb_AA': {'range': [0, np.pi]}, + 'dphi_Pb_Cb_BB': {'range': [0, np.pi]}, + 'dphi_Pb_Cb_AB': {'range': [0, np.pi]}, + 'dphi_Pb_Cb_BA': {'range': [0, np.pi]}, + } SelectedFields = [ # all observables @@ -109,21 +162,6 @@ ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], - - # adjusted recursive jigsaw reconstruction - ['mP', 'mC', 'mX', - 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', - 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', - 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', - 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', - 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', - 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', - 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', - 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', - 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', - 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], ] Name = 'SUSYModel' @@ -138,8 +176,8 @@ 'arrType': float, 'WeightInitialization':"'normal'"} -Params = {'Width': [1024], - 'Depth': [2], +Params = {'Width': [1585], + 'Depth': [1], 'loss': ["'categorical_crossentropy'"]} PS = Permutator (Params) diff --git a/ZllConfig.py b/ZllConfig.py index e907131..34cb9d6 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -7,7 +7,7 @@ # Select datasets (formerly TTrees in original ROOT file) Samples = [] for InputData in InputFiles: - InputData = os.getenv('SampleDir') + '/' + InputData + InputData = os.getenv('SampleDirZll') + '/' + InputData Samples += [ [InputData, 'Zto2LOS'], [InputData, 'Rndm2LOS'] ] pass @@ -15,13 +15,15 @@ #Previously in InputVars.py # Select Variables To use in training Observables = { - # group similar - 'Group': [ - ['LP_pT', 'LP_E', - 'LM_pT', 'LM_E'], - ['LP_phi', 'LM_phi'], - ['LP_eta', 'LM_eta'], - ], + 'LP_pT': {'trim': [0, 99]}, + 'LP_phi': {'range': [-np.pi, np.pi]}, + 'LP_eta': {'range': [-2.5, 2.5]}, + 'LP_E': {'trim': [0, 99]}, + + 'LM_pT': {'trim': [0, 99]}, + 'LM_phi': {'range': [-np.pi, np.pi]}, + 'LM_eta': {'range': [-2.5, 2.5]}, + 'LM_E': {'trim': [0, 99]}, } SelectedFields = [ @@ -36,7 +38,7 @@ Name = 'ZllModel' Config = {'MaxEvents': 50000, - 'Epochs': 1000, + 'Epochs': 5000, 'BatchSize': 2048*8, 'LearningRate': 0.005, 'Decay': 0., From 99a17dcf08da0aa4e0a594262b9070ac1cb06b0c Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Tue, 10 Jan 2017 22:03:01 -0600 Subject: [PATCH 37/38] retest --- Experiment.py | 14 +++++++------- SUSYConfig.py | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Experiment.py b/Experiment.py index aeb92e0..d04f8e4 100644 --- a/Experiment.py +++ b/Experiment.py @@ -78,12 +78,12 @@ yy1 = Test_X[obs] yy1[:] = 1./(maxval-minval) * (yy1-minval) # gaus - yy = Train_X[obs] - yy1 = Train_X[obs] - M = np.mean (Train_X0[obs]) - V = np.var (Train_X0[obs]) - yy[:] = (yy - M) / V + .5 - yy1[:] = (yy1 - M) / V + .5 + #yy = Train_X[obs] + #yy1 = Train_X[obs] + #M = np.mean (Train_X0[obs]) + #V = np.var (Train_X0[obs]) + #yy[:] = (yy - M) / V + .5 + #yy1[:] = (yy1 - M) / V + .5 pass pass @@ -163,7 +163,7 @@ if WriteResults: print 'Writing Results.' from EventClassificationDNN.CSVWriter import * - CSVWriter (MyModel.OutDir+'/Result.csv', Test_X0, Test_Y, result, arrType) + CSVWriter (MyModel.OutDir+'/Result.csv', Test_X0, Test_Y, result, Config['arrType']) pass a = datetime.datetime.now() diff --git a/SUSYConfig.py b/SUSYConfig.py index 476212e..71bc948 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -29,9 +29,9 @@ # used for scaling Observables = { - 'mP': {}, - 'mC': {}, - 'mX': {}, + 'mP': {'range': [1000, 1001]}, + 'mC': {'range': [150, 950]}, + 'mX': {'range': [100, 101]}, 'B1_pT': {'trim': [0, 99]}, 'B1_phi': {'range': [-np.pi, np.pi]}, 'B1_eta': {'range': [-2.5, 2.5]}, @@ -137,7 +137,7 @@ 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M'], # physics reconstruction - ['mP', 'mC', 'mX', + ['mC', 'METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', From afe43848081ea4da5cec749093e293a6d1b4dacc Mon Sep 17 00:00:00 2001 From: dbullockphd Date: Sun, 15 Jan 2017 22:05:37 -0600 Subject: [PATCH 38/38] update bbll+MET --- Experiment.py | 70 ++++++----------- SUSYConfig.py | 212 +++++++++++++++++++++----------------------------- ZllConfig.py | 12 +-- 3 files changed, 116 insertions(+), 178 deletions(-) diff --git a/Experiment.py b/Experiment.py index d04f8e4..b0f6673 100644 --- a/Experiment.py +++ b/Experiment.py @@ -30,60 +30,34 @@ Train_X0 = Train_X.copy() Test_X0 = Test_X.copy() -#### Trimming -print 'Trimming outliers from data ...' -num = Train_X.shape[0] -for obs in Observables: - if not Observables[obs].has_key('trim'): continue - print ' ', obs - if Observables[obs]['trim'][0] < 0: lower = -np.inf - else: lower = np.nanpercentile(Train_X[obs], Observables[obs]['trim'][0]) - if Observables[obs]['trim'][1] > 100: upper = np.inf - else: upper = np.nanpercentile(Train_X[obs], Observables[obs]['trim'][1]) - for evt in xrange(num): - val = Train_X[obs][evt] - if np.isnan(val) or np.isinf(val) or np.isneginf(val): - Train_X[obs][evt] = np.nan - pass - elif val > upper: - Train_X[obs][evt] = np.nan - pass - elif val < lower: - Train_X[obs][val] = np.nan - pass - pass - pass -for evt in xrange(num): - if any(np.isnan(Train_X[obs][evt]) for obs in Observables): - for obs in Observables: - Train_X[obs][evt] = np.nan - pass - pass - pass - #### Shifting/Scaling normalization -print 'Normalizing', obs, '...' +print 'Normalizing ...' for obs in Observables: print ' ', obs - if Observables[obs].has_key('range'): - minval = Observables[obs]['range'][0] - maxval = Observables[obs]['range'][1] + yy = Train_X[obs] + yy1 = Test_X[obs] + if Observables[obs].has_key('gaus'): + M = np.mean(Train_X[obs]) + V = np.var(Test_X[obs]) + yy[:] = (yy - M) / V + 0.5 + yy1[:] = (yy1 - M) / V + 0.5 pass else: - minval = np.min(Train_X[obs]) - maxval = np.max(Train_X[obs]) + if Observables[obs].has_key('trim'): + minval = np.nanpercentile(Train_X[obs], Observables[obs]['trim'][0]) + maxval = np.nanpercentile(Train_X[obs], Observables[obs]['trim'][1]) + pass + elif Observables[obs].has_key('range'): + minval = Observables[obs]['range'][0] + maxval = Observables[obs]['range'][1] + pass + else: + minval = np.min(Train_X[obs]) + maxval = np.max(Train_X[obs]) + pass + yy[:] = 1./(maxval-minval) * (yy-minval) + yy1[:] = 1./(maxval-minval) * (yy1-minval) pass - yy = Train_X[obs] - yy[:] = 1./(maxval-minval) * (yy-minval) - yy1 = Test_X[obs] - yy1[:] = 1./(maxval-minval) * (yy1-minval) - # gaus - #yy = Train_X[obs] - #yy1 = Train_X[obs] - #M = np.mean (Train_X0[obs]) - #V = np.var (Train_X0[obs]) - #yy[:] = (yy - M) / V + .5 - #yy1[:] = (yy1 - M) / V + .5 pass pass diff --git a/SUSYConfig.py b/SUSYConfig.py index 71bc948..9d11222 100644 --- a/SUSYConfig.py +++ b/SUSYConfig.py @@ -3,16 +3,16 @@ #Previously in InputFiles.py # Define the Input Files InputFiles = ['mP1000_mC150_mX100.h5', - 'mP1000_mC400_mX100.h5', - 'mP1000_mC600_mX100.h5', - 'mP1000_mC900_mX100.h5', 'mP1000_mC200_mX100.h5', - 'mP1000_mC500_mX100.h5', - 'mP1000_mC700_mX100.h5', - 'mP1000_mC950_mX100.h5', 'mP1000_mC300_mX100.h5', + 'mP1000_mC400_mX100.h5', + 'mP1000_mC500_mX100.h5', 'mP1000_mC550_mX100.h5', - 'mP1000_mC800_mX100.h5'] + 'mP1000_mC600_mX100.h5', + 'mP1000_mC700_mX100.h5', + 'mP1000_mC800_mX100.h5', + 'mP1000_mC900_mX100.h5', + 'mP1000_mC950_mX100.h5'] # Select datasets (formerly TTrees in original ROOT file) Samples = [] @@ -29,106 +29,86 @@ # used for scaling Observables = { - 'mP': {'range': [1000, 1001]}, - 'mC': {'range': [150, 950]}, - 'mX': {'range': [100, 101]}, - 'B1_pT': {'trim': [0, 99]}, - 'B1_phi': {'range': [-np.pi, np.pi]}, - 'B1_eta': {'range': [-2.5, 2.5]}, - 'B1_M': {'trim': [0, 99]}, - 'B2_pT': {'trim': [0, 99]}, - 'B2_phi': {'range': [-np.pi, np.pi]}, - 'B2_eta': {'range': [-2.5, 2.5]}, - 'B2_M': {'trim': [0, 99]}, - 'L1_pT': {'trim': [0, 99]}, - 'L1_phi': {'range': [-np.pi, np.pi]}, - 'L1_eta': {'range': [-2.5, 2.5]}, - 'L1_M': {'trim': [0, 99]}, - 'L2_pT': {'trim': [0, 99]}, - 'L2_phi': {'range': [-np.pi, np.pi]}, - 'L2_eta': {'range': [-2.5, 2.5]}, - 'L2_M': {'trim': [0, 99]}, - 'METx': {'trim': [0, 99]}, - 'METy': {'trim': [0, 99]}, - 'MPP_AA': {}, - 'Eb_a_AA': {}, - 'Eb_b_AA': {}, - 'El_a_AA': {}, - 'El_b_AA': {}, - 'MPP_BB': {}, - 'Eb_a_BB': {}, - 'Eb_b_BB': {}, - 'El_a_BB': {}, - 'El_b_BB': {}, - 'MPP_AB': {}, - 'Eb_a_AB': {}, - 'Eb_b_AB': {}, - 'El_a_AB': {}, - 'El_b_AB': {}, - 'MPP_BA': {}, - 'Eb_a_BA': {}, - 'Eb_b_BA': {}, - 'El_a_BA': {}, - 'El_b_BA': {}, - 'cosPP_AA': {'range': [0, 1]}, - 'cosPP_AB': {'range': [0, 1]}, - 'cosPP_BA': {'range': [0, 1]}, - 'cosPP_BB': {'range': [0, 1]}, - 'cosPa_AA': {'range': [0, 1]}, - 'cosPa_AB': {'range': [0, 1]}, - 'cosPa_BA': {'range': [0, 1]}, - 'cosPa_BB': {'range': [0, 1]}, - 'cosPb_AA': {'range': [0, 1]}, - 'cosPb_AB': {'range': [0, 1]}, - 'cosPb_BA': {'range': [0, 1]}, - 'cosPb_BB': {'range': [0, 1]}, - 'cosCa_AA': {'range': [0, 1]}, - 'cosCa_AB': {'range': [0, 1]}, - 'cosCa_BA': {'range': [0, 1]}, - 'cosCa_BB': {'range': [0, 1]}, - 'cosCb_AA': {'range': [0, 1]}, - 'cosCb_AB': {'range': [0, 1]}, - 'cosCb_BA': {'range': [0, 1]}, - 'cosCb_BB': {'range': [0, 1]}, - 'dphi_PP_Pa_AA': {'range': [0, np.pi]}, - 'dphi_PP_Pa_BB': {'range': [0, np.pi]}, - 'dphi_PP_Pa_AB': {'range': [0, np.pi]}, - 'dphi_PP_Pa_BA': {'range': [0, np.pi]}, - 'dphi_PP_Pb_AA': {'range': [0, np.pi]}, - 'dphi_PP_Pb_BB': {'range': [0, np.pi]}, - 'dphi_PP_Pb_AB': {'range': [0, np.pi]}, - 'dphi_PP_Pb_BA': {'range': [0, np.pi]}, - 'dphi_Pa_Ca_AA': {'range': [0, np.pi]}, - 'dphi_Pa_Ca_BB': {'range': [0, np.pi]}, - 'dphi_Pa_Ca_AB': {'range': [0, np.pi]}, - 'dphi_Pa_Ca_BA': {'range': [0, np.pi]}, - 'dphi_Pb_Cb_AA': {'range': [0, np.pi]}, - 'dphi_Pb_Cb_BB': {'range': [0, np.pi]}, - 'dphi_Pb_Cb_AB': {'range': [0, np.pi]}, - 'dphi_Pb_Cb_BA': {'range': [0, np.pi]}, + 'mP': {'range': [1000, 1001]}, + 'mC': {'range': [150, 950]}, + 'mX': {'range': [100, 101]}, + 'B1_pT': {'trim': [0, 99]}, + 'B1_phi': {'range': [-np.pi, np.pi]}, + 'B1_eta': {'range': [-2.5, 2.5]}, + 'B1_M': {'trim': [0, 99]}, + 'B2_pT': {'trim': [0, 99]}, + 'B2_phi': {'range': [-np.pi, np.pi]}, + 'B2_eta': {'range': [-2.5, 2.5]}, + 'B2_M': {'trim': [0, 99]}, + 'L1_pT': {'trim': [0, 99]}, + 'L1_phi': {'range': [-np.pi, np.pi]}, + 'L1_eta': {'range': [-2.5, 2.5]}, + 'L1_M': {'trim': [0, 99]}, + 'L2_pT': {'trim': [0, 99]}, + 'L2_phi': {'range': [-np.pi, np.pi]}, + 'L2_eta': {'range': [-2.5, 2.5]}, + 'L2_M': {'trim': [0, 99]}, + 'METx': {'trim': [0, 99]}, + 'METy': {'trim': [0, 99]}, + 'MPP_AA': {}, + 'Eb_a_AA': {}, + 'Eb_b_AA': {}, + 'El_a_AA': {}, + 'El_b_AA': {}, + 'MPP_BB': {}, + 'Eb_a_BB': {}, + 'Eb_b_BB': {}, + 'El_a_BB': {}, + 'El_b_BB': {}, + 'MPP_AB': {}, + 'Eb_a_AB': {}, + 'Eb_b_AB': {}, + 'El_a_AB': {}, + 'El_b_AB': {}, + 'MPP_BA': {}, + 'Eb_a_BA': {}, + 'Eb_b_BA': {}, + 'El_a_BA': {}, + 'El_b_BA': {}, + 'cosPP_AA': {'range': [-1, 1]}, + 'cosPP_AB': {'range': [-1, 1]}, + 'cosPP_BA': {'range': [-1, 1]}, + 'cosPP_BB': {'range': [-1, 1]}, + 'cosPa_AA': {'range': [-1, 1]}, + 'cosPa_AB': {'range': [-1, 1]}, + 'cosPa_BA': {'range': [-1, 1]}, + 'cosPa_BB': {'range': [-1, 1]}, + 'cosPb_AA': {'range': [-1, 1]}, + 'cosPb_AB': {'range': [-1, 1]}, + 'cosPb_BA': {'range': [-1, 1]}, + 'cosPb_BB': {'range': [-1, 1]}, + 'cosCa_AA': {'range': [-1, 1]}, + 'cosCa_AB': {'range': [-1, 1]}, + 'cosCa_BA': {'range': [-1, 1]}, + 'cosCa_BB': {'range': [-1, 1]}, + 'cosCb_AA': {'range': [-1, 1]}, + 'cosCb_AB': {'range': [-1, 1]}, + 'cosCb_BA': {'range': [-1, 1]}, + 'cosCb_BB': {'range': [-1, 1]}, + 'dphi_PP_Pa_AA': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pa_BB': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pa_AB': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pa_BA': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pb_AA': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pb_BB': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pb_AB': {'range': [0, 2.*np.pi]}, + 'dphi_PP_Pb_BA': {'range': [0, 2.*np.pi]}, + 'dphi_Pa_Ca_AA': {'range': [0, 2.*np.pi]}, + 'dphi_Pa_Ca_BB': {'range': [0, 2.*np.pi]}, + 'dphi_Pa_Ca_AB': {'range': [0, 2.*np.pi]}, + 'dphi_Pa_Ca_BA': {'range': [0, 2.*np.pi]}, + 'dphi_Pb_Cb_AA': {'range': [0, 2.*np.pi]}, + 'dphi_Pb_Cb_BB': {'range': [0, 2.*np.pi]}, + 'dphi_Pb_Cb_AB': {'range': [0, 2.*np.pi]}, + 'dphi_Pb_Cb_BA': {'range': [0, 2.*np.pi]}, } SelectedFields = [ - # all observables - ['mP', 'mC', 'mX', - 'METx', 'METy', - 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', - 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', - 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', - 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M', - 'MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', - 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', - 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', - 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', - 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', - 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', - 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', - 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', - 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', - 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], - # detector reconstruction ['METx', 'METy', 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', @@ -136,29 +116,13 @@ 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M'], - # physics reconstruction - ['mC', - 'METx', 'METy', - 'L1_pT', 'L1_eta', 'L1_phi', 'L1_M', - 'L2_pT', 'L2_eta', 'L2_phi', 'L2_M', - 'B1_pT', 'B1_eta', 'B1_phi', 'B1_M', - 'B2_pT', 'B2_eta', 'B2_phi', 'B2_M'], - - # recursive jigsaw reconstruction + # recursive jigsaw reconstruction, just mass/energy ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'MPP_AB', 'Eb_a_AB', 'Eb_b_AB', 'El_a_AB', 'El_b_AB', 'MPP_BA', 'Eb_a_BA', 'Eb_b_BA', 'El_a_BA', 'El_b_BA', - 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB', - 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', - 'cosPP_AB', 'cosPa_AB', 'cosPb_AB', 'cosCa_AB', 'cosCb_AB', - 'cosPP_BA', 'cosPa_BA', 'cosPb_BA', 'cosCa_BA', 'cosCb_BA', - 'cosPP_BB', 'cosPa_BB', 'cosPb_BB', 'cosCa_BB', 'cosCb_BB', - 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA', - 'dphi_PP_Pa_AB', 'dphi_PP_Pb_AB', 'dphi_Pa_Ca_AB', 'dphi_Pb_Cb_AB', - 'dphi_PP_Pa_BA', 'dphi_PP_Pb_BA', 'dphi_Pa_Ca_BA', 'dphi_Pb_Cb_BA', - 'dphi_PP_Pa_BB', 'dphi_PP_Pb_BB', 'dphi_Pa_Ca_BB', 'dphi_Pb_Cb_BB'], - - # single topology reconstruction + 'MPP_BB', 'Eb_a_BB', 'Eb_b_BB', 'El_a_BB', 'El_b_BB'], + + # single topologic reconstruction ['MPP_AA', 'Eb_a_AA', 'Eb_b_AA', 'El_a_AA', 'El_b_AA', 'cosPP_AA', 'cosPa_AA', 'cosPb_AA', 'cosCa_AA', 'cosCb_AA', 'dphi_PP_Pa_AA', 'dphi_PP_Pb_AA', 'dphi_Pa_Ca_AA', 'dphi_Pb_Cb_AA'], diff --git a/ZllConfig.py b/ZllConfig.py index 34cb9d6..0915acc 100644 --- a/ZllConfig.py +++ b/ZllConfig.py @@ -15,15 +15,15 @@ #Previously in InputVars.py # Select Variables To use in training Observables = { - 'LP_pT': {'trim': [0, 99]}, + 'LP_pT': {'trim': [0, 99]}, 'LP_phi': {'range': [-np.pi, np.pi]}, 'LP_eta': {'range': [-2.5, 2.5]}, - 'LP_E': {'trim': [0, 99]}, + 'LP_E': {'trim': [0, 99]}, - 'LM_pT': {'trim': [0, 99]}, + 'LM_pT': {'trim': [0, 99]}, 'LM_phi': {'range': [-np.pi, np.pi]}, 'LM_eta': {'range': [-2.5, 2.5]}, - 'LM_E': {'trim': [0, 99]}, + 'LM_E': {'trim': [0, 99]}, } SelectedFields = [ @@ -44,13 +44,13 @@ 'Decay': 0., 'Momentum': 0., 'Nesterov': 0., + 'arrType' = np.float32, 'WeightInitialization':"'normal'"} -Params = {'Width': [1585], +Params = {'Width': [1585, 100], 'Depth': [1], 'loss': ["'categorical_crossentropy'"]} -arrType = np.float32 PS = Permutator (Params) Combos = PS.Permutations ()