Added Option to Tune on Test Set & Added Comments

Author: SuyashLakhotia

misc/tune_linearsvc.py

@@ -29,6 +29,10 @@
 parser.add_argument("--out", type=str, default="tfidf", choices=["tfidf", "count"],
                     help="Type of document vectors (default: tfidf)")
 
+parser.add_argument("--test", action="store_false", dest="validation",
+                    help="Include this flag if models should be tuned on the test set instead.")
+parser.set_defaults(validation=True)
+
 parser.add_argument("--min", type=float, default=0)
 parser.add_argument("--max", type=float, default=5)
 
@@ -45,10 +49,13 @@
 y_train = train.labels
 y_test = test.labels
 
-# Split training set & validation set
-validation_index = -1 * int(0.1 * float(len(y_train)))
-x_train, x_valid = x_train[:validation_index], x_train[validation_index:]
-y_train, y_valid = y_train[:validation_index], y_train[validation_index:]
+if args.validation:
+    # Split training set & validation set
+    validation_index = -1 * int(0.1 * float(len(y_train)))
+    x_train, x_valid = x_train[:validation_index], x_train[validation_index:]
+    y_train, y_valid = y_train[:validation_index], y_train[validation_index:]
+else:
+    x_valid, y_valid = [], []
 
 # Print information about the dataset
 print("")
@@ -69,29 +76,42 @@
 # Training
 # ==================================================
 
-acc_dict = {}
+# Generate C values to test [min, max, 0.1]
 C_arr = [float('%.1f' % i) for i in np.arange(args.min, args.max + 0.1, 0.1)]
+
+# Train & test models with different hyperparameter values
+acc_dict = {}
 for i in C_arr:
     if i <= 0:
         continue
     svm_clf = LinearSVC(C=i)
     svm_clf.fit(x_train, y_train)
-    predicted = svm_clf.predict(x_valid)
-    svm_acc = np.mean(predicted == y_valid)
+    if args.validation:
+        predicted = svm_clf.predict(x_valid)
+        svm_acc = np.mean(predicted == y_valid)
+    else:
+        predicted = svm_clf.predict(x_test)
+        svm_acc = np.mean(predicted == y_test)
     acc_dict[i] = svm_acc
     print("C {:.2f}: {:g}".format(i, svm_acc))
 
 print(acc_dict)
 print("")
 
-x_train = vstack((x_train, x_valid))
-y_train = np.concatenate((y_train, y_valid), axis=0)
+# Concatenate training set & validation set to form original train set
+if args.validation:
+    x_train = vstack((x_train, x_valid))
+    y_train = np.concatenate((y_train, y_valid), axis=0)
+
+# Get optimized hyperparameter
 max_C = max(acc_dict.keys(), key=(lambda key: acc_dict[key]))
 
+# Re-train & test model with chosen hyperparameter
 svm_clf = LinearSVC(C=max_C)
 svm_clf.fit(x_train, y_train)
 predicted = svm_clf.predict(x_test)
 svm_acc = np.mean(predicted == y_test)
 
+# Print result of final model
 utils.print_result(args.dataset, "linear_svc", svm_acc, data_str, str(int(time.time())),
                    hyperparams="{{C: {}}}".format(max_C))