Merge pull request #25 from Yamato-Security/2.0.0-dev

[WIP] 2.0.0 dev

.gitignore

@@ -145,6 +145,9 @@ dmypy.json
 #Takajo binaries
 takajo
 takajo*.exe
+takajo-*
 
 #Results files
-*.txt
+*.txt
+output/*
+*.xlsx

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changes
 
+## v2.0.0 [2022/08/03]
+
+- Major update released at the [SANS DFIR Summit in Austin](https://www.sans.org/cyber-security-training-events/digital-forensics-summit-2023/).
+
+**New Features:**
+
+- `list-domains`: create a list of unique domains (input: JSONL, profile: standard) (@YamatoSecurity)
+- `list-hashes`: create a list of process hashes to be used with vt-hash-lookup (input: JSONL, profile: standard) (@YamatoSecurity)
+- `list-ip-addresses`: create a list of unique target and/or source IP addresses (input: JSONL, profile: standard) (@YamatoSecurity)
+- `split-csv-timeline`: split up a large CSV file into smaller ones based on the computer name (input: non-multiline CSV, profile: any) (@YamatoSecurity)
+- `split-json-timeline`: split up a large JSONL timeline into smaller ones based on the computer name (input: JSONL, profile: any) (@fukusuket)
+- `stack-logons`: stack logons by target user, target computer, source IP address and source computer (input: JSONL, profile: standard) (@YamatoSecurity)
+- `sysmon-process-tree`: output the process tree of a certain process (input: JSONL, profile: standard) (@hitenkoku)
+- `timeline-logon`: create a CSV timeline of logon events (input: JSONL, profile: standard) (@YamatoSecurity)
+- `timeline-suspicious-processes`: create a CSV timeline of suspicious processes (input: JSONL, profile: standard) (@YamatoSecurity)
+- `vt-domain-lookup`: look up a list of domains on VirusTotal (input: text file) (@YamatoSecurity)
+- `vt-hash-lookup`: look up a list of hashes on VirusTotal (input: text file) (@YamatoSecurity)
+- `vt-ip-lookup`: look up a list of IP addresses on VirusTotal (input: text file) (@YamatoSecurity)
+
 ## v1.0.0 [2022/10/28]
 
 - Official release at [Code Blue 2022 Bluebox](https://codeblue.jp/2022/en/talks/?content=talks_24).

src/takajo.nim

@@ -1,130 +1,226 @@
+import algorithm
 import cligen
-import os
+import json
+import httpclient
+import sets
+import sequtils
+import strformat
+import strutils
+import tables
 import terminal
-import std/sequtils
-import std/strformat
-import std/strutils
-import std/tables
-import takajopkg/submodule
-
-proc listUndetectedEvtxFiles(timeline: string, evtxDir: string,
-    columnName: system.string = "EvtxFile", quiet: bool = false,
-        output: string = ""): int =
-
-  if not quiet:
-    styledEcho(fgGreen, outputLogo())
-
-  let csvData: TableRef[string, seq[string]] = getHayabusaCsvData(timeline, columnName)
-  var fileLists: seq[string] = getTargetExtFileLists(evtxDir, ".evtx")
-
-  var detectedPaths: seq[string] = csvData[columnName].map(getFileNameWithExt)
-  detectedPaths = deduplicate(detectedPaths)
-
-  let checkResult = getUnlistedSeq(fileLists, detectedPaths)
-  var outputStock: seq[string] = @[]
-
-  echo "Finished. "
-  echo "---------------"
-
-  if checkResult.len == 0:
-    echo "Great! No undetected evtx files were found."
-    echo ""
-  else:
-    echo "Undetected evtx file identified."
-    echo ""
-    var numberOfEvtxFiles = 0
-    for undetectedFile in checkResult:
-      outputStock.add(undetectedFile)
-      inc numberOfEvtxFiles
-    outputStock.add("")
-    let undetectedPercentage = (checkResult.len() / fileLists.len()) * 100
-    echo fmt"{ undetectedPercentage :.4}% of the evtx files did not have any detections."
-    echo fmt"Number of evtx files not detected: {numberOfEvtxFiles}"
-    echo ""
-  if output != "":
-    let f = open(output, fmWrite)
-    defer: f.close()
-    for line in outputStock:
-      f.writeLine(line)
-    echo fmt"Saved File {output}"
-    echo ""
-  else:
-    echo outputstock.join("\n")
-  discard
-
-
-proc listUnusedRules(timeline: string, rulesDir: string,
-    columnName: string = "RuleFile", quiet: bool = false,
-        output: string = ""): int =
-
-  if not quiet:
-    styledEcho(fgGreen, outputLogo())
-
-  let csvData: TableRef[string, seq[string]] = getHayabusaCsvData(timeline, columnName)
-  var fileLists: seq[string] = getTargetExtFileLists(rulesDir, ".yml")
-  var detectedPaths: seq[string] = csvData[columnName].map(getFileNameWithExt)
-  detectedPaths = deduplicate(detectedPaths)
-  var outputStock: seq[string] = @[]
-
-  echo "Finished. "
-  echo "---------------"
-
-  let checkResult = getUnlistedSeq(fileLists, detectedPaths)
-  if checkResult.len == 0:
-    echo "Great! No unused rule files were found."
-    echo ""
-  else:
-    echo "Unused rule file identified."
-    echo ""
-    var numberOfUnusedRules = 0
-    for undetectedFile in checkResult:
-      outputStock.add(undetectedFile)
-      inc numberOfUnusedRules
-    let undetectedPercentage = (checkResult.len() / fileLists.len()) * 100
-    outputStock.add("")
-    echo fmt"{ undetectedPercentage :.4}% of the yml rules were not used."
-    echo fmt"Number of unused rule files: {numberOfUnusedRules}"
-    echo ""
-  if output != "":
-    let f = open(output, fmWrite)
-    defer: f.close()
-    for line in outputStock:
-      f.writeLine(line)
-    echo fmt"Saved File {output}"
-    echo ""
-  else:
-    echo outputstock.join("\n")
-  discard
+import times
+import uri
+import os
+import std/enumerate
+import suru
+import takajopkg/general
+include takajopkg/listDomains
+include takajopkg/listIpAddresses
+include takajopkg/listUndetectedEvtxFiles
+include takajopkg/listUnusedRules
+include takajopkg/splitCsvTimeline
+include takajopkg/splitJsonTimeline
+include takajopkg/stackLogons
+include takajopkg/listHashes
+include takajopkg/sysmonProcessTree
+include takajopkg/timelineLogon
+include takajopkg/timelineSuspiciousProcesses
+include takajopkg/vtDomainLookup
+include takajopkg/vtIpLookup
+include takajopkg/vtHashLookup
 
 when isMainModule:
-  clCfg.version = "1.0.0-dev"
-  const examples = "Examples:\n  undetected-evtx -t ../hayabusa/timeline.csv -e ../hayabusa-sample-evtx\n  unused-rules -t ../hayabusa/timeline.csv -r ../hayabusa/rules\n"
-  clCfg.useMulti = "Usage: takajo.exe <COMMAND>\n\nCommands:\n$subcmds\nCommand help: $command help <COMMAND>\n\n" & examples
-
-  if paramCount() == 0:
-    styledEcho(fgGreen, outputLogo())
-  dispatchMulti(
-    [
-      listUndetectedEvtxFiles, cmdName = "undetected-evtx",
-      doc = "List up undetected evtx files",
-      help = {
-        "timeline": "CSV timeline created by Hayabusa with verbose profile",
-        "evtxDir": "The directory of .evtx files you scanned with Hayabusa",
-        "columnName": "Specify custom column header name",
-        "quiet": "Do not display the launch banner",
-        "output": "Save the results to a text file. The default is to print to screen.",
-      }
-    ],
-    [
-      listUnusedRules, cmdName = "unused-rules",
-      doc = "List up unused rules",
-      help = {
-        "timeline": "CSV timeline created by Hayabusa with verbose profile",
-        "rulesDir": "Hayabusa rules directory",
-        "columnName": "Specify custom column header name",
-        "quiet": "Do not display the launch banner",
-        "output": "Save the results to a text file. The default is to print to screen.",
-      }
-    ]
-  )
-
+    clCfg.version = "2.0.0"
+    const examples = "Examples:\p"
+    const example_list_domains = "  list-domains -t ../hayabusa/timeline.jsonl -o domains.txt\p"
+    const example_list_ip_addresses = "  list-ip-addresses -t ../hayabusa/timeline.jsonl -o ipAddresses.txt\p"
+    const example_list_undetected_evtx = "  list-undetected-evtx -t ../hayabusa/timeline.csv -e ../hayabusa-sample-evtx\p"
+    const example_list_unused_rules = "  list-unused-rules -t ../hayabusa/timeline.csv -r ../hayabusa/rules\p"
+    const example_split_csv_timeline = "  split-csv-timeline -t ../hayabusa/timeline.csv [--makeMultiline] -o case-1-csv\p"
+    const example_split_json_timeline = "  split-json-timeline -t ../hayabusa/timeline.jsonl -o case-1-json\p"
+    const example_stack_logons = "  stack-logons -t ../hayabusa/timeline.jsonl -o logons.csv\p"
+    const example_list_hashes = "  list-hashes -t ../hayabusa/case-1.jsonl -o case-1\p"
+    const example_sysmon_process_tree = "  sysmon-process-tree -t ../hayabusa/timeline.jsonl -p <Process GUID> [-o process-tree.txt]\p"
+    const example_timeline_logon = "  timeline-logon -t ../hayabusa/timeline.jsonl -o logon-timeline.csv\p"
+    const example_timeline_suspicious_processes = "  timeline-suspicious-processes -t ../hayabusa/timeline.jsonl [--level medium] [-o suspicious-processes.csv]\p"
+    const example_vt_domain_lookup = "  vt-domain-lookup  -a <API-KEY> --domainList domains.txt -r 1000 -o results.csv --jsonOutput responses.json\p"
+    const example_vt_hash_lookup = "  vt-hash-lookup -a <API-KEY> --hashList case-1-MD5-hashes.txt -r 1000 -o results.csv --jsonOutput responses.json\p"
+    const example_vt_ip_lookup = "  vt-ip-lookup -a <API-KEY> --ipList ipAddresses.txt -r 1000 -o results.csv --jsonOutput responses.json\p"
+
+    clCfg.useMulti = "Version: 2.0.0\pUsage: takajo.exe <COMMAND>\p\pCommands:\p$subcmds\pCommand help: $command help <COMMAND>\p\p" &
+        examples & example_list_domains & example_list_hashes & example_list_ip_addresses & example_list_undetected_evtx & example_list_unused_rules &
+        example_split_csv_timeline & example_split_json_timeline & example_stack_logons & example_sysmon_process_tree &
+        example_timeline_logon & example_timeline_suspicious_processes &
+        example_vt_domain_lookup & example_vt_hash_lookup & example_vt_ip_lookup
+
+    if paramCount() == 0:
+        styledEcho(fgGreen, outputLogo())
+    dispatchMulti(
+        [
+            listDomains, cmdName = "list-domains",
+            doc = "create a list of unique domains to be used with vt-domain-lookup",
+            help = {
+                "includeSubdomains": "include subdomains",
+                "includeWorkstations": "include local workstation names",
+                "output": "save results to a text file",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            },
+            short = {
+                "includeSubdomains": 's',
+                "includeWorkstations": 'w'
+            }
+        ],
+        [
+            listHashes, cmdName = "list-hashes",
+            doc = "create a list of process hashes to be used with vt-hash-lookup",
+            help = {
+                "level": "specify the minimum alert level",
+                "output": "specify the base name to save results to text files (ex: -o case-1)",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            }
+        ],
+        [
+            listIpAddresses, cmdName = "list-ip-addresses",
+            doc = "create a list of unique target and/or source IP addresses to be used with vt-ip-lookup",
+            help = {
+                "inbound": "include inbound traffic",
+                "outbound": "include outbound traffic",
+                "output": "save results to a text file",
+                "privateIp": "include private IP addresses",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            },
+            short = {
+                "output": 'o',
+                "outbound": 'O'
+            }
+        ],
+        [
+            listUndetectedEvtxFiles, cmdName = "list-undetected-evtx",
+            doc = "create a list of undetected evtx files",
+            help = {
+                "columnName": "specify a custom column header name",
+                "evtxDir": "directory of .evtx files you scanned with Hayabusa",
+                "output": "save the results to a text file (default: stdout)",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa CSV timeline (profile: any verbose profile)",
+            }
+        ],
+        [
+            listUnusedRules, cmdName = "list-unused-rules",
+            doc = "create a list of unused sigma rules",
+            help = {
+                "columnName": "specify a custom column header name",
+                "output": "save the results to a text file (default: stdout)",
+                "quiet": "do not display the launch banner",
+                "rulesDir": "Hayabusa rules directory",
+                "timeline": "Hayabusa CSV timeline (profile: any verbose profile)",
+            }
+        ],
+        [
+            splitCsvTimeline, cmdName = "split-csv-timeline",
+            doc = "split up a large CSV file into smaller ones based on the computer name",
+            help = {
+                "makeMultiline": "output fields in multiple lines",
+                "output": "output directory (default: output)",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa non-multiline CSV timeline (profile: any)",
+            }
+        ],
+        [
+            splitJsonTimeline, cmdName = "split-json-timeline",
+            doc = "split up a large JSONL timeline into smaller ones based on the computer name",
+            help = {
+                "output": "output directory (default: output)",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any)",
+            }
+        ],
+        [
+            stackLogons, cmdName = "stack-logons",
+            doc = "stack logons by target user, target computer, source IP address and source computer",
+            help = {
+                "localSrcIpAddresses": "include results when the source IP address is local",
+                "output": "save results to a CSV file",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            }
+        ],
+        [
+            sysmonProcessTree, cmdName = "sysmon-process-tree",
+            doc = "output the process tree of a certain process",
+            help = {
+                "output": "save results to a text file",
+                "processGuid": "sysmon process GUID",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            }
+        ],
+        [
+            timelineLogon, cmdName = "timeline-logon",
+            doc = "create a CSV timeline of logon events",
+            help = {
+                "calculateElapsedTime": "calculate the elapsed time for successful logons",
+                "output": "save results to a CSV file",
+                "outputAdminLogonEvents": "output admin logon events as separate entries",
+                "outputLogoffEvents": "output logoff events as separate entries",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            },
+            short = {
+                "outputLogoffEvents": 'l',
+                "outputAdminLogonEvents": 'a'
+            }
+        ],
+        [
+            timelineSuspiciousProcesses, cmdName = "timeline-suspicious-processes",
+            doc = "create a CSV timeline of suspicious processes",
+            help = {
+                "level": "specify the minimum alert level",
+                "output": "save results to a CSV file",
+                "quiet": "do not display the launch banner",
+                "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)",
+            }
+        ],
+        [
+            vtDomainLookup, cmdName = "vt-domain-lookup",
+            doc = "look up a list of domains on VirusTotal",
+            help = {
+                "apiKey": "your VirusTotal API key",
+                "domainList": "a text file list of domains",
+                "jsonOutput": "save all responses to a JSON file",
+                "output": "save results to a CSV file",
+                "rateLimit": "set the rate per minute for requests",
+                "quiet": "do not display the launch banner",
+            }
+        ],
+        [
+            vtHashLookup, cmdName = "vt-hash-lookup",
+            doc = "look up a list of hashes on VirusTotal",
+            help = {
+                "apiKey": "your VirusTotal API key",
+                "hashList": "a text file list of hashes",
+                "jsonOutput": "save all responses to a JSON file",
+                "output": "save results to a text file",
+                "rateLimit": "set the rate per minute for requests",
+                "quiet": "do not display the launch banner",
+            },
+            short = {
+                "hashList": 'H'
+            }
+        ],
+        [
+            vtIpLookup, cmdName = "vt-ip-lookup",
+            doc = "look up a list of IP addresses on VirusTotal",
+            help = {
+                "apiKey": "your VirusTotal API key",
+                "ipList": "a text file list of IP addresses",
+                "jsonOutput": "save all responses to a JSON file",
+                "output": "save results to a CSV file",
+                "rateLimit": "set the rate per minute for requests",
+                "quiet": "do not display the launch banner",
+            }
+        ]
+    )