Move all containers to autokit

Makefile

@@ -15,6 +15,7 @@ UPARGS := --force-recreate --remove-orphans
 QEMUCOMPOSEFILE := docker-compose.qemu.yml
 SECUREBOOTCOMPOSEFILE := docker-compose.secureboot.yml
 CLIENTCOMPOSEFILE := docker-compose.client.yml
+WORKERCOMPOSEFILE := docker-compose.worker.yml
 
 # only use the qemu compose file if worker type is qemu
 ifeq ($(WORKER_TYPE),qemu)
@@ -23,12 +24,14 @@ ifeq ($(QEMU_SECUREBOOT),1)
 COMPOSE_FILE := $(COMPOSE_FILE):$(SECUREBOOTCOMPOSEFILE)
 endif
 else
-COMPOSE_FILE := $(CLIENTCOMPOSEFILE)
+COMPOSE_FILE := $(WORKERCOMPOSEFILE)
 endif
 
 # for arm64 hosts we need to set the BALENA_ARCH to pull the correct balenalib worker image
 ifeq ($(shell uname -m),aarch64)
 BALENA_ARCH ?= aarch64
+else ifeq ($(shell uname -m),armv7l)
+BALENA_ARCH ?= armv7hf
 else ifeq ($(shell uname -m),arm64)
 BALENA_ARCH ?= aarch64
 else
@@ -58,6 +61,8 @@ $(DOCKERCOMPOSE):
 	mkdir -p $(shell dirname "$@")
 ifeq ($(shell uname -m),arm64)
 	curl -fsSL "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(shell uname -s | tr '[:upper:]' '[:lower:]')-aarch64" -o $@
+else ifeq ($(shell uname -m),armv7l)
+	curl -fsSL "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(shell uname -s | tr '[:upper:]' '[:lower:]')-armv7" -o $@
 else
 	curl -fsSL "https://github.com/docker/compose/releases/download/v2.3.3/docker-compose-$(shell uname -s | tr '[:upper:]' '[:lower:]')-$(shell uname -m)" -o $@
 endif
@@ -83,6 +88,9 @@ test: $(DOCKERCOMPOSE) build ## Run the test suites
 local-test: ## Alias for 'make test WORKER_TYPE=qemu'
 	$(MAKE) test WORKER_TYPE=qemu
 
+local-test-autokit: ## Alias for 'make test WORKER_TYPE=qemu'
+	$(MAKE) test WORKER_TYPE=autokit
+
 qemu: ## Alias for 'make test WORKER_TYPE=qemu'
 	$(MAKE) test WORKER_TYPE=qemu
 

client/Dockerfile

@@ -17,4 +17,4 @@ VOLUME /usr/src/app/reports
 
 ENTRYPOINT ["/usr/src/app/entry.sh"]
 
-CMD [ "-c", "/usr/src/app/workspace/config" ]
+CMD [ "-c", "/data/workspace/config" ]

core/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14.19.3-bullseye-slim as base
+FROM node:16-bullseye-slim as base
 
 WORKDIR /usr/app
 
@@ -8,23 +8,20 @@ ENV DEBIAN_FRONTEND noninteractive
 # install docker, balena-cli dependencies, and suite dependencies
 # https://github.com/balena-io/balena-cli/blob/master/INSTALL-LINUX.md#additional-dependencies
 # hadolint ignore=DL3008
+
 RUN apt-get update && apt-get install --no-install-recommends -y \
-	bind9-dnsutils \
 	ca-certificates \
+	wget \
+	bind9-dnsutils \
 	docker.io \
-	git \
 	iproute2 \
-	jq \
 	openssh-client \
-	socat \
-	rsync \
+	libudev-dev \	
 	unzip \
 	util-linux \
-	wget \
-	vim \
 	build-essential \
 	make \
-	python && \
+	python3 && \
 	apt-get clean && \
 	rm -rf /var/lib/apt/lists/*
 
@@ -36,19 +33,26 @@ RUN if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ] ; \
 	then \
 		wget -q -O balena-cli.zip "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_REF}/balena-cli-v${BALENA_CLI_VERSION}-linux-arm64-standalone.zip" && \
 		unzip balena-cli.zip && rm balena-cli.zip ; \
+	elif [ "$(uname -m)" = "armv7l" ] || [ "$(uname -m)" = "armv7hf" ] ; \
+	then \
+		npm i balena-cli@latest ; \
+		mkdir /usr/app/cli/ ; \
+		mv /usr/app/node_modules/balena-cli/ /usr/app ; \
+		rm -rf /usr/app/node_modules/ ; \
 	else \
 		wget -q -O balena-cli.zip "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_REF}/balena-cli-v${BALENA_CLI_VERSION}-linux-x64-standalone.zip" && \
 		unzip balena-cli.zip && rm balena-cli.zip ; \
 	fi
 
 # Add balena-cli to PATH
 ENV PATH /usr/app/balena-cli:$PATH
+# ENV PATH $PATH:/usr/app/balena-cli/bin
 
 RUN balena version
 
 COPY package*.json ./
 
-RUN npm ci
+RUN npm ci --only=production
 
 COPY . .
 

core/balena.yml

@@ -0,0 +1,8 @@
+name: leviathan-core
+type: sw.block
+description: 'Leviathan-core is the testrunner for leviathan'
+assets:
+  repository:
+    type: blob.asset
+    data:
+      url: 'https://github.com/balena-os/leviathan'

core/entry.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 
+# Internet doesn't work in docker-in-docker when in network_mode: host 
+# due to nftables and iptables-legacy conflict. Docker creates rules in
+# iptables-legacy which is different from what the host (nftables) uses 
+# leading to containers without internet. Commands need to run before starting Docker
+# 
+# Check out: https://stackoverflow.com/a/76488849/8522689
+
+echo "Changing iptables to legacy iptables"
+update-alternatives --set iptables /usr/sbin/iptables-legacy
+update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
 rm -rf /var/run/docker 2>/dev/null || true
 rm -f /var/run/docker.sock 2>/dev/null || true
 rm -f /var/run/docker.pid 2>/dev/null || true

core/lib/common/worker.js

@@ -95,6 +95,7 @@ module.exports = class Worker {
 		this.directConnect = (
 			this.url.includes(`worker`)
 			|| this.url.includes('unix:')
+			|| this.url.includes('localhost')
 		);
 		if (this.url.includes(`balena-devices.com`)) {
 			// worker is a testbot connected to balena cloud - we ssh into it via the vpn
@@ -121,56 +122,14 @@ module.exports = class Worker {
 			async () => {
 				attempt++;
 				this.logger.log(`Preparing to flash, attempt ${attempt}...`);
-
-				await new Promise(async (resolve, reject) => {
-					const req = rp.post({ uri: `${this.url}/dut/flash` });
-
-					req.catch((error) => {
-						reject(error);
-					});
-					req.finally(() => {
-						if (lastStatus !== 'done') {
-							reject(new Error('Unexpected end of TCP connection'));
-						}
-
-						resolve();
-					});
-
-					let lastStatus;
-					req.on('data', (data) => {
-						const computedLine = RegExp('(.+?): (.*)').exec(data.toString());
-
-						if (computedLine) {
-							if (computedLine[1] === 'error') {
-								req.cancel();
-								reject(new Error(computedLine[2]));
-							}
-
-							if (computedLine[1] === 'progress') {
-								once(() => {
-									this.logger.log('Flashing');
-								});
-								// Hide any errors as the lines we get can be half written
-								const state = JSON.parse(computedLine[2]);
-								if (state != null && isNumber(state.percentage)) {
-									this.logger.status({
-										message: 'Flashing',
-										percentage: state.percentage,
-									});
-								}
-							}
-
-							if (computedLine[1] === 'status') {
-								lastStatus = computedLine[2];
-							}
-						}
-					});
-
-					pipeline(
-						fs.createReadStream(imagePath),
-						createGzip({ level: 6 }),
-						req,
-					);
+				this.logger.log(`Image path being sent: ${imagePath}`)
+				await doRequest({
+					method: 'POST',
+					uri: `${this.url}/dut/flash`,
+					body: {
+						path: imagePath
+					},
+					json: true,
 				});
 				this.logger.log('Flash completed');
 			},
@@ -416,12 +375,8 @@ module.exports = class Worker {
 	async executeCommandInWorker(command, retryOptions={}) {
 		return retry(
 			async () => {
-				let containerId = await this.executeCommandInWorkerHost(
-					`balena ps | grep worker | awk '{print $1}'`,
-				);
-				let result = await this.executeCommandInWorkerHost(
-					`balena exec ${containerId} ${command}`,
-				);
+				let result = await exec(command);
+				console.log(`Exec call: ${command}, Result: ${result}`)
 				return result;
 			},
 			{

core/lib/config.js

@@ -1,12 +1,11 @@
 module.exports = {
   leviathan: {
     artifacts: '/tmp/artifacts',    // To store artifacts meant to be reported as results at the end of the suite
-    downloads: '/data/downloads',   // To store/download assets needed for the suite (non-persistent) 
+    downloads: '/data/workspace/downloads',   // To store/download assets needed for the suite (non-persistent) 
     reports: '/reports/',           // To store/download reports generated from the suite (non-persistent) 
     workdir: '/data',
     uploads: {
-      image: '/data/os.img',
-      config: '/data/config.json',
+      config:'/data/config.json',
       suite: '/data/suite'
     }
   }

docker-compose.client.yml

@@ -4,9 +4,12 @@ services:
   client:
     build: ./client
     volumes:
-      - "${WORKSPACE:-./workspace}:/usr/src/app/workspace:ro"
-      - "${REPORTS:-./workspace/reports}:/usr/src/app/reports:rw"
-      - "${SUITES:-./suites}:/usr/src/app/suites:ro"
+      # - "${WORKSPACE:-./workspace}:/usr/src/app/workspace:ro"
+      # - "${REPORTS:-./workspace/reports}:/usr/src/app/reports:rw"
+      # - "${SUITES:-./suites}:/usr/src/app/suites:ro"
+      - "${SUITES:-./suites}:/data/suites"
+      - "${REPORTS:-./workspace/reports}:/data/reports"
+      - "${WORKSPACE:-./workspace}:/data/workspace"  
     environment:
       - WORKER_TYPE=${WORKER_TYPE}
       - DEVICE_TYPE=${DEVICE_TYPE}

docker-compose.worker.yml

@@ -0,0 +1,60 @@
+version: "2"
+
+services:
+  client:
+    build: ./client
+    network_mode: host
+    volumes:
+      # - "${WORKSPACE:-./workspace}:/usr/src/app/workspace:ro"
+      # - "${REPORTS:-./workspace/reports}:/usr/src/app/reports:rw"
+      # - "${SUITES:-./suites}:/usr/src/app/suites:ro"
+      - "${SUITES:-./suites}:/data/suites"
+      - "${REPORTS:-./workspace/reports}:/data/reports"
+      - "${WORKSPACE:-./workspace}:/data/workspace"  
+    environment:
+      - WORKER_TYPE=${WORKER_TYPE}
+      - DEVICE_TYPE=${DEVICE_TYPE}
+      - BALENACLOUD_API_KEY=${BALENACLOUD_API_KEY}
+      - BALENACLOUD_ORG=${BALENACLOUD_ORG}
+      - BALENACLOUD_APP_NAME=${BALENACLOUD_APP_NAME}
+      - CORE_HOST=localhost
+    depends_on:
+      - core
+
+  core:
+    privileged: true # preload requires docker-in-docker
+    #build: core
+    image: bh.cr/gh_rcooke_warwick/leviathan-core-armv7hf
+    network_mode: host
+    volumes:
+      - reports-storage:/reports
+      - "${SUITES:-./suites}:/data/suites"
+      - "${REPORTS:-./workspace/reports}:/data/reports"
+      - "${WORKSPACE:-./workspace}:/data/workspace"
+    tmpfs:
+      - /var/run # use tmpfs docker-in-docker pid files
+      - /var/lib/docker # use tmpfs for docker-in-docker data root
+    restart: 'no'
+
+  worker:
+    image: bh.cr/gh_rcooke_warwick/leviathan-worker-amrv7hf
+    privileged: true
+    pid: host
+    network_mode: host
+    ipc: host
+    volumes:
+      - 'reports-storage:/reports'
+      - "${SUITES:-./suites}:/data/suites"
+      - "${REPORTS:-./workspace/reports}:/data/reports"
+      - "${WORKSPACE:-./workspace}:/data/workspace"
+      - /host/run/dbus:/host/run/dbus
+      - /lib/modules:/lib/modules
+    environment:
+      - UDEV=1
+      - BALENA_API_KEY=${BALENA_API_KEY}
+      - WORKER_TYPE=autokit
+      - DIGITAL_RELAY=dummyPower
+      - TESTBOT_DUT_TYPE=raspberrypi4-64
+
+volumes:
+  reports-storage:

testrunner/Dockerfile

@@ -0,0 +1,18 @@
+# avoid alpine 3.13 or later due to this issue on armv7
+# https://wiki.alpinelinux.org/wiki/Release_Notes_for_Alpine_3.13.0#time64_requirements
+
+FROM balenalib/armv7hf-alpine:3.15-run 
+
+WORKDIR /usr/src/app
+
+# hadolint ignore=DL3018
+RUN apk add --no-cache git ca-certificates docker nano make curl
+
+# fail if binaries are missing or won't run
+RUN dockerd --version && docker --version
+
+# install entrypoint script
+COPY entry.sh ./
+
+RUN chmod +x entry.sh
+CMD [ "/usr/src/app/entry.sh" ]

testrunner/docker-compose.yml

@@ -0,0 +1,23 @@
+version: "2.1"
+
+volumes:
+  docker:
+
+services:
+  main:
+    build: .
+    privileged: true
+    pid: host
+    network_mode: host
+    ipc: host
+    volumes:
+      - docker:/var/lib/docker
+    tmpfs:
+      - /tmp
+      #- /var/run
+      - /var/log
+    cpu_shares: 512
+    labels:
+      io.balena.features.dbus: '1'
+      io.balena.features.balena-api: '1'
+      io.balena.features.kernel-modules: '1'

testrunner/entry.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Clone Leviathan
+https://github.com/balena-os/Leviathan
+
+rm -rf /var/run/docker 2>/dev/null || true
+rm -f /var/run/docker.sock 2>/dev/null || true
+rm -f /var/run/docker.pid 2>/dev/null || true
+
+dockerd &
+
+tail -f /dev/null