From 5b032cdf93ffb10e7dd9105030d2c28917990359 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 12 Oct 2022 11:17:39 +0200 Subject: [PATCH] [5946] add job in circleci, quick scan script fix --- .circleci/config.yml | 68 ++++++++++++++++++++++++++++++++++++++++++++ Makefile | 6 ++++ scans/zapscan.py | 63 ++++++++++++++++++++++++---------------- 3 files changed, 112 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f94b920580b..36cf207a39c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -261,6 +261,66 @@ jobs: fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade semgrep && make -C .. semgrep" + zap-vulnerability-scan: + machine: + image: ubuntu-2004:202010-01 + enabled: true + environment: + DOCKER_API_VERSION: 1.23 + BASE_OS: focal + parallelism: 3 + steps: + - checkout + - *rebaseontarget + - *createcachedir + - *restorecache + - *loadimagelayers + - *dockerimagebuild + - *saveimagelayers + - *savecache + + - run: + name: Install dependencies + command: | + sudo systemctl stop apt-daily.service + sudo systemctl kill --kill-who=all apt-daily.service + while ! (systemctl list-units --all apt-daily.service | egrep -q '(dead|failed)') do sleep 1; done + ( sudo apt-get update || sudo apt-get update ) + sudo apt-get install -y openjdk-17-jre-headless wget firefox + export GECKODRIVER_VER=v0.30.0 + wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VER}/geckodriver-${GECKODRIVER_VER}-linux64.tar.gz -O /tmp/geckodriver.tar.gz + cd /tmp + tar -xvzf geckodriver.tar.gz + sudo install geckodriver /usr/local/bin + wget https://github.com/zaproxy/zaproxy/releases/download/v2.11.1/ZAP_2_11_1_unix.sh -O /tmp/zap_installer.sh + chmod u+x /tmp/zap_installer.sh + sudo /tmp/zap_installer.sh -q + zap.sh -cmd -addoninstall jython + cd ~/project; ls + pip3 install -r scans/requirements.txt + + - run: + name: Run dev instance + command: | + fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}') + DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make dev-detatched + background: true + + - run: + name: Run zap daemon + command: zap.sh -daemon -port 8090 -config api.disablekey=true -config hud.enabled=false -config hud.enabledForDesktop=false + background: true + + - run: + name: Run zap + command: python3 ~/project/scans/zapscan.py + + - store_test_results: + path: ~/project/jrn_report.html + + - store_artifacts: + path: ~/project/src_report.html + staging-test-with-rebase: machine: image: ubuntu-2004:202010-01 @@ -367,6 +427,14 @@ workflows: - /update-builder-.*/ requires: - lint + - zap-vulnerability-scan: + requires: + - lint + filters: + branches: + ignore: + - /i18n-.*/ + - /update-builder-.*/ nightly: triggers: diff --git a/Makefile b/Makefile index 18a0a86404d..d310bc36bfb 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,12 @@ dev: ## Run the development server in a Docker container. @echo "███ Starting development server..." @OFFSET_PORTS='false' DOCKER_BUILD_VERBOSE='true' $(DEVSHELL) $(SDBIN)/run @echo + +.PHONY: dev-detatched +dev-detatched: ## Run the development server in a Docker container without attatching tty. + @echo "███ Starting development server..." + @OFFSET_PORTS='false' DETATCHED='true' DOCKER_BUILD_VERBOSE='true' $(DEVSHELL) $(SDBIN)/run + @echo .PHONY: dev-tor dev-tor: ## Run the development server with onion services in a Docker container. diff --git a/scans/zapscan.py b/scans/zapscan.py index 2caae4efcf3..46f907f595c 100644 --- a/scans/zapscan.py +++ b/scans/zapscan.py @@ -1,6 +1,6 @@ from time import sleep import pyotp -from subprocess import run +from subprocess import run, CalledProcessError from enum import Enum from selenium.webdriver import Firefox, FirefoxOptions @@ -9,7 +9,8 @@ from selenium.common.exceptions import WebDriverException, NoSuchElementException -# Test credentials from https://developers.securedrop.org/en/latest/setup_development.html#using-the-docker-environment +# Test credentials from docs +# https://developers.securedrop.org/en/latest/setup_development.html#using-the-docker-environment SOURCE_URL = "http://127.0.0.1:8080" JOURNALIST_URL = "http://127.0.0.1:8081" @@ -20,6 +21,7 @@ SCAN_CMD_FMT = "zap-cli active-scan {url}" REPORT_CMD_FMT = "zap-cli report -f {cmd_ftype} -o {filename}" + class ReportType(Enum): XML = 1 HTML = 2 @@ -50,7 +52,7 @@ def prepare_source_iface(base_url: str, driver: Firefox): generate_url = base_url + "/generate" driver.get(generate_url) elem = driver.find_element(By.ID, "codename") - codename = elem.text + # codename = elem.text continue_btn = driver.find_element(By.ID, "create-form").find_element(By.TAG_NAME, "button") continue_btn.click() @@ -80,20 +82,24 @@ def export_report(outfile="zap_report.html", filetype=ReportType.HTML): cmd_ftype = "xml" elif filetype == ReportType.MARKDOWN: cmd_ftype = "md" - else: raise ValueError("filetype is not one of: ReportType.HTML, ReportType.XML, ReportType.MARKDOWN") + else: + raise ValueError("filetype is not one of: ReportType.HTML, ReportType.XML, ReportType.MARKDOWN") cmdstr = REPORT_CMD_FMT.format(cmd_ftype=cmd_ftype, filename=outfile) - res = run(cmdstr, shell=True, check=True) - return res.returncode + try: + run(cmdstr, shell=True, check=True) + except: + print("Failed to write report to file: {}".format(outfile)) + raise -def run_zap_scan(base_url: str, outfile="report.html") -> bool: +def run_zap_scan(base_url: str, outfile="report.html"): cmdstr = SCAN_CMD_FMT.format(url=base_url) - res = run(cmdstr, shell=True) - if res.returncode != 0: - return False - if export_report(outfile=outfile) != 0: - return False - return True + try: + run(cmdstr, shell=True, check=True) + export_report(outfile=outfile) + except: + print("Zap scan failed for {}, with reporting in file {}".format(base_url, outfile)) + raise def scan(base_url: str, login_fn=None, report_file="report.html"): @@ -102,7 +108,9 @@ def scan(base_url: str, login_fn=None, report_file="report.html"): sleep(2) if login_fn: login_fn(base_url, driver) - run_zap_scan(base_url, outfile=report_file) + try: + run_zap_scan(base_url, outfile=report_file) + except: raise driver.quit() @@ -155,21 +163,26 @@ def wait_for_services(): def main(): wait_for_services() print("Starting scan of journalist interface") - jrn_res = scan(JOURNALIST_URL, login_fn=prepare_journalist_iface, report_file="jrn_report.html") - if jrn_res: + jrn_failed, src_failed = False, False + try: + scan(JOURNALIST_URL, login_fn=prepare_journalist_iface, report_file="jrn_report.html") print("Journalist interface scan complete") print("Starting scan of source interface") - else: - print("Journalist interface scan encountered an error; proceeding to source interface scan") - src_res = scan(SOURCE_URL, login_fn=prepare_source_iface, report_file="src_report.html") - if jrn_res: + except Exception as e: + jrn_failed = True + print("Scan failed for journalist interface, trying source interface...") + print(e) + try: + scan(SOURCE_URL, login_fn=prepare_source_iface, report_file="src_report.html") print("Source interface scan complete") - else: + except Exception as e: + src_failed = True print("Source interface scan encountered an error") - if not src_res or not jrn_res: - if not jrn_res: print("Journalist interface failed to complete") - if not src_res: print("Source interface failed to complete") - exit(1) + print(e) + if jrn_failed: + print("Journalist interface failed to complete") + if src_failed: + print("Source interface failed to complete") if __name__ == "__main__":