[5946] add job in circleci, quick scan script fix

freedomofpress · Oct 19, 2022 · d789ec4 · d789ec4
1 parent 6bf8bad
commit d789ec4
Show file tree

Hide file tree

Showing 3 changed files with 116 additions and 27 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -261,6 +261,66 @@ jobs:
             fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
             DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade semgrep && make -C .. semgrep"
 
+  zap-vulnerability-scan:
+    machine:
+      image: ubuntu-2004:202010-01
+      enabled: true
+    environment:
+      DOCKER_API_VERSION: 1.23
+      BASE_OS: focal
+    parallelism: 3
+    steps:
+      - checkout
+      - *rebaseontarget
+      - *createcachedir
+      - *restorecache
+      - *loadimagelayers
+      - *dockerimagebuild
+      - *saveimagelayers
+      - *savecache
+
+      - run:
+          name: Install dependencies
+          command: |
+            sudo systemctl stop apt-daily.service
+            sudo systemctl kill --kill-who=all apt-daily.service
+            while ! (systemctl list-units --all apt-daily.service | egrep -q '(dead|failed)') do sleep 1; done
+            ( sudo apt-get update || sudo apt-get update )
+            sudo apt-get install -y openjdk-17-jre-headless wget firefox
+            export GECKODRIVER_VER=v0.30.0
+            wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VER}/geckodriver-${GECKODRIVER_VER}-linux64.tar.gz -O /tmp/geckodriver.tar.gz
+            cd /tmp
+            tar -xvzf geckodriver.tar.gz
+            sudo install geckodriver /usr/local/bin
+            wget https://github.com/zaproxy/zaproxy/releases/download/v2.11.1/ZAP_2_11_1_unix.sh -O /tmp/zap_installer.sh
+            chmod u+x /tmp/zap_installer.sh
+            sudo /tmp/zap_installer.sh -q
+            zap.sh -cmd -addoninstall jython
+            cd ~/project; ls
+            pip3 install -r scans/requirements.txt
+
+      - run:
+          name: Run dev instance
+          command: |
+            fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make dev-detatched
+          background: true
+
+      - run:
+          name: Run zap daemon
+          command: zap.sh -daemon -port 8090 -config api.disablekey=true -config hud.enabled=false -config hud.enabledForDesktop=false
+          background: true
+
+      - run:
+          name: Run zap
+          command: python3 ~/project/scans/zapscan.py
+
+      - store_test_results:
+          path: ~/project/jrn_report.html
+
+      - store_artifacts:
+          path: ~/project/src_report.html
+
   staging-test-with-rebase:
     machine:
       image: ubuntu-2004:202010-01
@@ -367,6 +427,14 @@ workflows:
                 - /update-builder-.*/
           requires:
             - lint
+      - zap-vulnerability-scan:
+          requires:
+            - lint
+          filters:
+            branches:
+              ignore:
+                - /i18n-.*/
+                - /update-builder-.*/
 
   nightly:
     triggers:

diff --git a/Makefile b/Makefile
@@ -237,6 +237,12 @@ dev:  ## Run the development server in a Docker container.
 	@echo "███ Starting development server..."
 	@OFFSET_PORTS='false' DOCKER_BUILD_VERBOSE='true' $(DEVSHELL) $(SDBIN)/run
 	@echo
+
+.PHONY: dev-detatched
+dev-detatched:  ## Run the development server in a Docker container without attatching tty.
+	@echo "███ Starting development server..."
+	@OFFSET_PORTS='false' DETATCHED='true' DOCKER_BUILD_VERBOSE='true' $(DEVSHELL) $(SDBIN)/run
+	@echo
 
 .PHONY: dev-tor
 dev-tor:  ## Run the development server with onion services in a Docker container.

diff --git a/scans/zapscan.py b/scans/zapscan.py
@@ -9,7 +9,8 @@
 from selenium.common.exceptions import WebDriverException, NoSuchElementException
 
 
-# Test credentials from https://developers.securedrop.org/en/latest/setup_development.html#using-the-docker-environment
+# Test credentials from docs
+# https://developers.securedrop.org/en/latest/setup_development.html#using-the-docker-environment
 
 SOURCE_URL = "http://127.0.0.1:8080"
 JOURNALIST_URL = "http://127.0.0.1:8081"
@@ -20,13 +21,15 @@
 SCAN_CMD_FMT = "zap-cli active-scan {url}"
 REPORT_CMD_FMT = "zap-cli report -f {cmd_ftype} -o {filename}"
 
+
 class ReportType(Enum):
     XML = 1
     HTML = 2
     MARKDOWN = 3
 
 
-class ServiceNotUpException(Exception): pass
+class ServiceNotUpException(Exception):
+    pass
 
 
 def get_ff_options(proxy_addr="127.0.0.1:8090") -> FirefoxOptions:
@@ -49,8 +52,8 @@ def start_driver() -> Firefox():
 def prepare_source_iface(base_url: str, driver: Firefox):
     generate_url = base_url + "/generate"
     driver.get(generate_url)
-    elem = driver.find_element(By.ID, "codename")
-    codename = elem.text
+    # elem = driver.find_element(By.ID, "codename")
+    # codename = elem.text
     continue_btn = driver.find_element(By.ID, "create-form").find_element(By.TAG_NAME, "button")
     continue_btn.click()
 
@@ -80,20 +83,24 @@ def export_report(outfile="zap_report.html", filetype=ReportType.HTML):
         cmd_ftype = "xml"
     elif filetype == ReportType.MARKDOWN:
         cmd_ftype = "md"
-    else: raise ValueError("filetype is not one of: ReportType.HTML, ReportType.XML, ReportType.MARKDOWN")
+    else:
+        raise ValueError("type is not one of: ReportType.HTML, ReportType.XML, ReportType.MARKDOWN")
     cmdstr = REPORT_CMD_FMT.format(cmd_ftype=cmd_ftype, filename=outfile)
-    res = run(cmdstr, shell=True, check=True)
-    return res.returncode
+    try:
+        run(cmdstr, shell=True, check=True)
+    except Exception:
+        print("Failed to write report to file: {}".format(outfile))
+        raise
 
 
-def run_zap_scan(base_url: str, outfile="report.html") -> bool:
+def run_zap_scan(base_url: str, outfile="report.html"):
     cmdstr = SCAN_CMD_FMT.format(url=base_url)
-    res = run(cmdstr, shell=True)
-    if res.returncode != 0:
-        return False
-    if export_report(outfile=outfile) != 0:
-        return False
-    return True
+    try:
+        run(cmdstr, shell=True, check=True)
+        export_report(outfile=outfile)
+    except Exception:
+        print("Zap scan failed for {}, with reporting in file {}".format(base_url, outfile))
+        raise
 
 
 def scan(base_url: str, login_fn=None, report_file="report.html"):
@@ -102,7 +109,10 @@ def scan(base_url: str, login_fn=None, report_file="report.html"):
     sleep(2)
     if login_fn:
         login_fn(base_url, driver)
-    run_zap_scan(base_url, outfile=report_file)
+    try:
+        run_zap_scan(base_url, outfile=report_file)
+    except Exception:
+        raise
     driver.quit()
 
 
@@ -155,22 +165,27 @@ def wait_for_services():
 def main():
     wait_for_services()
     print("Starting scan of journalist interface")
-    jrn_res = scan(JOURNALIST_URL, login_fn=prepare_journalist_iface, report_file="jrn_report.html")
-    if jrn_res:
+    jrn_failed, src_failed = False, False
+    try:
+        scan(JOURNALIST_URL, login_fn=prepare_journalist_iface, report_file="jrn_report.html")
         print("Journalist interface scan complete")
         print("Starting scan of source interface")
-    else:
-        print("Journalist interface scan encountered an error; proceeding to source interface scan")
-    src_res = scan(SOURCE_URL, login_fn=prepare_source_iface, report_file="src_report.html")
-    if jrn_res:
+    except Exception as e:
+        jrn_failed = True
+        print("Scan failed for journalist interface, trying source interface...")
+        print(e)
+    try:
+        scan(SOURCE_URL, login_fn=prepare_source_iface, report_file="src_report.html")
         print("Source interface scan complete")
-    else:
+    except Exception as e:
+        src_failed = True
         print("Source interface scan encountered an error")
-    if not src_res or not jrn_res:
-        if not jrn_res: print("Journalist interface failed to complete")
-        if not src_res: print("Source interface failed to complete")
-        exit(1)
+        print(e)
+    if jrn_failed:
+        print("Journalist interface failed to complete")
+    if src_failed:
+        print("Source interface failed to complete")
 
 
 if __name__ == "__main__":
-    main()
+    main()