Untangle tests (#112)

* Isolate S3 test case

* Isolate webdav test case

* Isolate SSH test case

* Isolate local storage test case

* Isolate gpg test case

* Add missing volume mount

* Fix file locations for local test case

* Remove compose test case, use utils

* Use test utils throughout

* Use dedicated tmp dir

* Fix link location that is being tested

* Use dedicated tmp_dirs when working on host fs

* Force delete artifact

* Fix expected filename

* Provide helpful messages on failing tests

* Fix filename

* Use proper volume names

* Fix syntax error, use large resource class

* Use named Docker volumes when referencing them in test scripts

* Add name of test case to logging output

.circleci/config.yml

@@ -5,6 +5,7 @@ jobs:
     machine:
       image: ubuntu-2004:202201-02
     working_directory: ~/docker-volume-backup
+    resource_class: large
     steps:
       - checkout
       - run:
@@ -29,6 +30,7 @@ jobs:
           DOCKER_BUILDKIT: '1'
           DOCKER_CLI_EXPERIMENTAL: enabled
     working_directory: ~/docker-volume-backup
+    resource_class: large
     steps:
       - checkout
       - setup_remote_docker:

test/cli/run.sh

@@ -3,6 +3,8 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
 
 docker network create test_network
 docker volume create backup_data
@@ -50,17 +52,11 @@ docker run --rm -it \
   -v backup_data:/data alpine \
   ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data'
 
-echo "[TEST:PASS] Found relevant files in untared remote backup."
+pass "Found relevant files in untared remote backup."
 
 # This test does not stop containers during backup. This is happening on
 # purpose in order to cover this setup as well.
-if [ "$(docker ps -q | wc -l)" != "2" ]; then
-  echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
-  docker ps
-  exit 1
-fi
-
-echo "[TEST:PASS] All containers running post backup."
+expect_running_containers "2"
 
 docker rm $(docker stop minio offen)
 docker volume rm backup_data app_data

test/commands/run.sh

@@ -3,7 +3,8 @@
 set -e
 
 cd $(dirname $0)
-
+. ../util.sh
+current_test=$(basename $(pwd))
 
 docker-compose up -d
 sleep 30 # mariadb likes to take a bit before responding
@@ -13,29 +14,27 @@ sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' commands_archive
 
 tar -xvf ./local/test.tar.gz
 if [ ! -f ./backup/data/dump.sql ]; then
-  echo "[TEST:FAIL] Could not find file written by pre command."
-  exit 1
+  fail "Could not find file written by pre command."
 fi
-echo "[TEST:PASS] Found expected file."
+pass "Found expected file."
 
 if [ -f ./backup/data/post.txt ]; then
-  echo "[TEST:FAIL] File created in post command was present in backup."
-  exit 1
+  fail "File created in post command was present in backup."
 fi
-echo "[TEST:PASS] Did not find unexpected file."
+pass "Did not find unexpected file."
 
 docker-compose down --volumes
 sudo rm -rf ./local
 
 
-echo "[TEST:INFO] Running commands test in swarm mode next."
+info "Running commands test in swarm mode next."
 
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
 
 while [ -z $(docker ps -q -f name=backup) ]; do
-  echo "[TEST:INFO] Backup container not ready yet. Retrying."
+  info "Backup container not ready yet. Retrying."
   sleep 1
 done
 
@@ -47,16 +46,14 @@ sudo cp -r $(docker volume inspect --format='{{ .Mountpoint }}' test_stack_archi
 
 tar -xvf ./local/test.tar.gz
 if [ ! -f ./backup/data/dump.sql ]; then
-  echo "[TEST:FAIL] Could not find file written by pre command."
-  exit 1
+  fail "Could not find file written by pre command."
 fi
-echo "[TEST:PASS] Found expected file."
+pass "Found expected file."
 
 if [ -f ./backup/data/post.txt ]; then
-  echo "[TEST:FAIL] File created in post command was present in backup."
-  exit 1
+  fail "File created in post command was present in backup."
 fi
-echo "[TEST:PASS] Did not find unexpected file."
+pass "Did not find unexpected file."
 
 docker stack rm test_stack
 docker swarm leave --force

test/confd/run.sh

@@ -3,6 +3,8 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
 
 mkdir -p local
 
@@ -14,19 +16,16 @@ sleep 100
 docker-compose down --volumes
 
 if [ ! -f ./local/conf.tar.gz ]; then
-  echo "[TEST:FAIL] Config from file was not used."
-  exit 1
+  fail "Config from file was not used."
 fi
-echo "[TEST:PASS] Config from file was used."
+pass "Config from file was used."
 
 if [ ! -f ./local/other.tar.gz ]; then
-  echo "[TEST:FAIL] Run on same schedule did not succeed."
-  exit 1
+  fail "Run on same schedule did not succeed."
 fi
-echo "[TEST:PASS] Run on same schedule succeeded."
+pass "Run on same schedule succeeded."
 
 if [ -f ./local/never.tar.gz ]; then
-  echo "[TEST:FAIL] Unexpected file was found."
-  exit 1
+  fail "Unexpected file was found."
 fi
-echo "[TEST:PASS] Unexpected cron did not run."
+pass "Unexpected cron did not run."

test/gpg/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    restart: always
+    environment:
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      GPG_PASSPHRASE: 1234secret
+    volumes:
+      - ./local:/archive
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/gpg/run.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker-compose up -d
+sleep 5
+
+docker-compose exec backup backup
+
+expect_running_containers "2"
+
+tmp_dir=$(mktemp -d)
+
+echo 1234secret | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test.tar.gz.gpg > ./local/decrypted.tar.gz
+tar -xf ./local/decrypted.tar.gz -C $tmp_dir
+ls -lah $tmp_dir
+if [ ! -f $tmp_dir/backup/app_data/offen.db ]; then
+  fail "Could not find expected file in untared archive."
+fi
+rm ./local/decrypted.tar.gz
+
+pass "Found relevant files in decrypted and untared local backup."
+
+if [ ! -L ./local/test-latest.tar.gz.gpg ]; then
+  fail "Could not find local symlink to latest encrypted backup."
+fi
+
+docker-compose down --volumes

test/ignore/run.sh

@@ -3,6 +3,9 @@
 set -e
 
 cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
 mkdir -p local
 
 docker-compose up -d
@@ -15,13 +18,11 @@ out=$(mktemp -d)
 sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
 
 if [ ! -f "$out/backup/data/me.txt" ]; then
-  echo "[TEST:FAIL] Expected file was not found."
-  exit 1
+  fail "Expected file was not found."
 fi
-echo "[TEST:PASS] Expected file was found."
+pass "Expected file was found."
 
 if [ -f "$out/backup/data/skip.me" ]; then
-  echo "[TEST:FAIL] Ignored file was found."
-  exit 1
+  fail "Ignored file was found."
 fi
-echo "[TEST:PASS] Ignored file was not found."
+pass "Ignored file was not found."

test/local/.gitignore

@@ -0,0 +1 @@
+local

test/local/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    hostname: hostnametoken
+    restart: always
+    environment:
+      BACKUP_FILENAME_EXPAND: 'true'
+      BACKUP_FILENAME: test-$$HOSTNAME.tar.gz
+      BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+      BACKUP_PRUNING_PREFIX: test
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./local:/archive
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  app_data:

test/local/run.sh

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+mkdir -p local
+
+docker-compose up -d
+sleep 5
+
+# A symlink for a known file in the volume is created so the test can check
+# whether symlinks are preserved on backup.
+docker-compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link
+docker-compose exec backup backup
+
+sleep 5
+
+expect_running_containers "2"
+
+tmp_dir=$(mktemp -d)
+tar -xvf ./local/test-hostnametoken.tar.gz -C $tmp_dir
+if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
+  fail "Could not find expected file in untared archive."
+fi
+rm -f ./local/test-hostnametoken.tar.gz
+
+if [ ! -L "$tmp_dir/backup/app_data/db.link" ]; then
+  fail "Could not find expected symlink in untared archive."
+fi
+
+pass "Found relevant files in decrypted and untared local backup."
+
+if [ ! -L ./local/test-hostnametoken.latest.tar.gz.gpg ]; then
+  fail "Could not find symlink to latest version."
+fi
+
+pass "Found symlink to latest version in local backup."
+
+# The second part of this test checks if backups get deleted when the retention
+# is set to 0 days (which it should not as it would mean all backups get deleted)
+# TODO: find out if we can test actual deletion without having to wait for a day
+BACKUP_RETENTION_DAYS="0" docker-compose up -d
+sleep 5
+
+docker-compose exec backup backup
+
+if [ "$(find ./local -type f | wc -l)" != "1" ]; then
+  fail "Backups should not have been deleted, instead seen: "$(find ./local -type f)""
+fi
+pass "Local backups have not been deleted."
+
+docker-compose down --volumes