classify flags to 'general', 'linear', 'nn' categories, and change some flags orders in main.py

Author: Winter Deng

dev/classifier.py

@@ -9,6 +9,29 @@
 lib_path = os.path.abspath(os.path.join(current_dir, "..", ".."))
 sys.path.insert(0, lib_path)
 
+import subprocess
+
+def load_exclude_authors(file_path):
+    path = Path(file_path)
+    if not path.exists():
+        return []
+    with open(path, "r", encoding="utf-8") as f:
+        return [line.strip() for line in f if line.strip()]
+
+def get_last_commit_excluding(exclude_file):
+    exclude_authors = set(load_exclude_authors(exclude_file))
+    
+    logs = subprocess.check_output([
+        "git", "log",
+        "--pretty=format:%H|%ae",  # commit hash | author email
+    ]).decode("utf-8").splitlines()
+    
+    for line in logs:
+        commit, email = line.split("|", 1)
+        if email not in exclude_authors:
+            return commit
+    return None
+
 def classify_file_category(path):
 
     relative_path = Path(path).relative_to(lib_path)
@@ -25,7 +48,6 @@ def classify_file_category(path):
 
 
 def fetch_option_flags(flags):
-    # flags = genflags.parser.flags
     flag_list = []
 
     for flag in flags:
@@ -42,6 +64,7 @@ def fetch_option_flags(flags):
 
 def fetch_all_files():
     main_files = [
+        os.path.join(lib_path, "main.py"),
         os.path.join(lib_path, "linear_trainer.py"),
         os.path.join(lib_path, "torch_trainer.py")
     ]
@@ -61,13 +84,27 @@ def find_config_usages_in_file(file_path, allowed_keys):
 
     category, path = classify_file_category(file_path)
 
+    if file_path.endswith("main.py"):
+        for idx in range(len(lines)):
+            if lines[idx].startswith("def main("):
+                lines = lines[idx:]
+                main_start = idx
+                break
+        for i, line in enumerate(lines[1:]):
+            if line and line[0] not in (" ", "\t") and line.strip() != "":
+                lines = lines[:i]
+                break
+
     for i, line in enumerate(lines, start=1):
         matches = pattern.findall(line)
         for key in matches:
             if key in allowed_keys:
                 if key not in detailed_results:
                     detailed_results[key] = {"file": path, "lines": []}
-                detailed_results[key]["lines"].append(str(i))
+                if file_path.endswith("main.py"):
+                    detailed_results[key]["lines"].append(str(i + main_start))
+                else:
+                    detailed_results[key]["lines"].append(str(i))
 
     return detailed_results
 
@@ -125,6 +162,7 @@ def classify(raw_flags):
     for flag in flags:
         if flag["category"] not in result:
             result[flag["category"]] = []
+
         result[flag["category"]].append({"name": flag["name"].replace("--", r"\-\-"), "description": flag["description"]})
 
     result["details"] = []
@@ -134,4 +172,4 @@ def classify(raw_flags):
             for i in v[1:]:
                 result["details"].append({"name": "", "file": i["file"], "location": ", ".join(i["lines"])})
 
-    return result
+    return result

dev/excel_tools.py

@@ -4,10 +4,8 @@
 sys.path.insert(1, os.path.join(sys.path[0], "..", ".."))
 
 import main
-
 from classifier import classify
 
-
 class FakeParser(dict):
     def __init__(self):
         self.flags = []
@@ -70,4 +68,4 @@ def print_table(title, flags, intro):
     classified["nn"],
     intro="**Neural network options**:\n\
 Configurations specific to torch (neural networks) trainer."
-)
+)