tests: Add end to end tests with bats

Author: rumpelsepp

.github/workflows/tests.yml

@@ -41,7 +41,28 @@ jobs:
         poetry install
     - name: Run pytest
       run: |
-        poetry run make test
+        poetry run make pytest
+
+  bats:
+    strategy:
+      fail-fast: false
+
+    runs-on: ubuntu-latest
+    container: debian:stable
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v3
+      with:
+        path: ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }}
+    - name: Install Dependencies
+      run: |
+        apt-get update -y && apt-get install -y bats python3 python3-poetry jq
+        poetry install
+    - name: Run bats
+      run: |
+        poetry run make bats
 
   vecu:
     strategy:

.shellcheckrc

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: CC0-1.0
+
+shell=bash

Makefile

@@ -10,7 +10,9 @@ default:
 	@echo " fmt         run autoformatters"
 	@echo " lint        run linters"
 	@echo " docs        build docs"
-	@echo " test        run testsuite"
+	@echo " tests       run testsuite"
+	@echo " pytest      run pytest tests"
+	@echo " bats        run bats end to end tests"
 	@echo " clean       delete build artifacts"
 
 .PHONY: zipapp
@@ -26,6 +28,7 @@ lint:
 	mypy src tests
 	ruff check src tests
 	ruff format --check src tests
+	find tests/bats \( -iname "*.bash" -or -iname "*.bats" \) | xargs shellcheck
 	reuse lint
 
 .PHONY: lint-win32
@@ -35,16 +38,24 @@ lint-win32:
 
 .PHONY: fmt
 fmt:
-	ruff check --fix-only src tests
-	ruff format src tests
+	ruff check --fix-only src tests/pytest
+	ruff format src tests/pytest
+	find tests/bats \( -iname "*.bash" -or -iname "*.bats" \) | xargs shfmt -w
 
 .PHONY: docs
 docs:
 	$(MAKE) -C docs html
 
-.PHONY: test
-test:
-	python -m pytest -v --cov=$(PWD) --cov-report html tests
+.PHONY: tests
+tests: pytest bats
+
+.PHONY: pytest
+pytest:
+	python -m pytest -v --cov=$(PWD) --cov-report html tests/pytest
+
+.PHONY: bats
+bats:
+	bats -x -r tests/bats
 
 .PHONY: clean
 clean:

flake.nix

@@ -14,6 +14,10 @@
       devShell.x86_64-linux = pkgs.mkShell {
         buildInputs = with pkgs; [ 
           poetry
+          shellcheck
+          shfmt
+          bats
+          nodePackages_latest.bash-language-server
           python311
           python312
         ];

tests/bats/001-invocation.bats

@@ -0,0 +1,10 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+@test "invoke gallia without parameters" {
+	# Should fail and print help page.
+	run -64 gallia
+}

tests/bats/002-scans.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load helpers.sh
+
+setup_file() {
+    setup_gallia_toml
+	run_vecu &
+}
+
+teardown_file() {
+    kill_vecu
+}
+
+@test "scan services" {
+	gallia scan uds services --sessions 1 2 --check-session
+}
+
+@test "scan sessions" {
+    gallia scan uds sessions --depth 2
+}
+
+@test "scan fast" {
+    gallia scan uds sessions --fast
+}
+
+@test "scan identifiers sid 0x22" {
+    gallia scan uds identifiers --start 0 --end 100 --sid 0x22
+}
+
+@test "scan identifiers sid 0x2e" {
+    gallia scan uds identifiers --start 0 --end 100 --sid 0x2e
+}
+
+@test "scan identifiers sid 0x31" {
+    gallia scan uds identifiers --start 0 --end 100 --sid 0x31
+}
+
+@test "scan reset" {
+    gallia scan uds reset
+}
+
+@test "scan dump-seeds" {
+    gallia scan uds dump-seeds --duration 0.01 --level 0x2f
+}
+
+@test "scan memory" {
+    for sid in 0x23 0x34 0x35 0x3d; do
+        poetry run gallia scan uds memory --sid "$sid"
+    done
+}

tests/bats/003-primitives.bats

@@ -0,0 +1,60 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load helpers.sh
+
+setup_file() {
+    setup_gallia_toml
+	run_vecu &
+}
+
+teardown_file() {
+    kill_vecu
+}
+
+@test "primitive ecu-reset" {
+    gallia primitive uds ecu-reset
+}
+
+@test "primitive vin" {
+    gallia primitive uds vin
+}
+
+@test "primitive ping" {
+    gallia primitive uds ping --count 2
+}
+
+@test "primitive rdbi" {
+    gallia primitive uds rdbi 0x108d
+}
+
+@test "primitive pdu" {
+    gallia primitive uds pdu 1001
+}
+
+@test "primitive wdbi" {
+    gallia primitive uds wdbi 0x2266 --data 00
+}
+
+@test "primitive dtc read" {
+    gallia primitive uds dtc read
+}
+
+@test "primitive dtc clear" {
+    gallia primitive uds dtc clear
+}
+
+@test "primitive dtc control stop" {
+    gallia primitive uds dtc control --stop
+}
+
+@test "primitive dtc control resume" {
+    gallia primitive uds dtc control --resume
+}
+
+@test "primitive iocbi reset-to-default" {
+    gallia primitive uds iocbi 0x1000 reset-to-default
+}

tests/bats/helpers.sh

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+close_non_std_fds() {
+    for fd in $(ls /proc/$BASHPID/fd); do
+        if [[ $fd -gt 2 ]]; then
+            eval "exec $fd>&-" || true
+        fi
+    done
+}
+
+run_vecu() {
+	close_non_std_fds
+    gallia script vecu "unix-lines:///tmp/vecu.sock" rng --seed 3 --mandatory_sessions "[1, 2, 3]" --mandatory_services "[DiagnosticSessionControl, EcuReset, ReadDataByIdentifier, WriteDataByIdentifier, RoutineControl, SecurityAccess, ReadMemoryByAddress, WriteMemoryByAddress, RequestDownload, RequestUpload, TesterPresent, ReadDTCInformation, ClearDiagnosticInformation, InputOutputControlByIdentifier]"
+}
+
+kill_vecu() {
+	kill -9 "$(pgrep gallia)"
+	rm -f /tmp/vecu.sock
+}
+
+setup_gallia_toml() {
+	{
+	    echo "[gallia]"
+	    echo "[gallia.scanner]"
+	    echo 'target = "unix-lines:///tmp/vecu.sock"'
+	    echo 'dumpcap = false'
+
+	    echo "[gallia.protocols.uds]"
+	    echo 'ecu_reset = 0x01'
+	} > "$BATS_FILE_TMPDIR/gallia.toml"
+
+	export GALLIA_CONFIG="$BATS_FILE_TMPDIR/gallia.toml"
+}
+

tests/bats/setup_suite.bash

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+setup_suite() {
+	bats_require_minimum_version 1.5.0
+
+	# https://bats-core.readthedocs.io/en/stable/tutorial.html#let-s-do-some-setup
+	DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" >/dev/null 2>&1 && pwd)"
+	PATH="$DIR/..:$PATH"
+
+	cd "$BATS_TEST_TMPDIR" || exit 1
+}