use ssh to send image instead of http to avoid timeouts

core/Dockerfile

@@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND noninteractive
 # https://github.com/balena-io/balena-cli/blob/master/INSTALL-LINUX.md#additional-dependencies
 # hadolint ignore=DL3008
 RUN apt-get update && apt-get install --no-install-recommends -y \
-	bind9-dnsutils \
+	bind9-dnsutils gzip\
 	ca-certificates \
 	docker.io \
 	git \

core/lib/common/worker.js

@@ -135,72 +135,60 @@ module.exports = class Worker {
 	 * @category helper
 	 */
 	async flash(imagePath) {
-		let attempt = 0;
-		await retry(
-			async () => {
-				attempt++;
-				this.logger.log(`Preparing to flash, attempt ${attempt}...`);
-
-				await new Promise(async (resolve, reject) => {
-					const req = rp.post({ uri: `${this.url}/dut/flash`, timeout: 0 });
-
-					req.catch((error) => {
-						this.logger.log(`client side error: `)
-						this.logger.log(error.message)
-						reject(error);
-					});
-					req.finally(() => {
-						if (lastStatus !== 'done') {
-							reject(new Error('Unexpected end of TCP connection'));
-						}
-
-						resolve();
-					});
-
-					let lastStatus;
-					req.on('data', (data) => {
-						const computedLine = RegExp('(.+?): (.*)').exec(data.toString());
-
-						if (computedLine) {
-							if (computedLine[1] === 'error') {
-								req.cancel();
-								reject(new Error(computedLine[2]));
-							}
-
-							if (computedLine[1] === 'progress') {
-								once(() => {
-									this.logger.log('Flashing');
-								});
-								// Hide any errors as the lines we get can be half written
-								const state = JSON.parse(computedLine[2]);
-								if (state != null && isNumber(state.percentage)) {
-									this.logger.status({
-										message: 'Flashing',
-										percentage: state.percentage,
-									});
-								}
-							}
-
-							if (computedLine[1] === 'status') {
-								lastStatus = computedLine[2];
-							}
-						}
-					});
-
-					pipeline(
-						fs.createReadStream(imagePath),
-						createGzip({ level: 6 }),
-						req,
-					);
-				});
-				this.logger.log('Flash completed');
-			},
-			{
-				max_tries: 5,
-				interval: 1000 * 5,
-				throw_original: true,
-			},
-		);
+
+		let TARGET_PATH = imagePath;
+		// if using remote worker i.e autokit/testbot, send the image over ssh
+		if(!this.directConnect){
+			this.logger.log(`Gzipping image...`);
+			await exec(`gzip ${imagePath}`);
+			// adjust image path to take new gz suffix
+			imagePath = `${imagePath}.gz`
+			// ensure we always send to the same location on worker to avoid overflowing of images
+			TARGET_PATH = `/data/os.img.gz`
+
+			// Wrap sending of image in a retry - hopefully the --partial arguement in the rsync command means it will resume
+			// in the case of an error
+			await retry(
+				async () => {
+					this.logger.log(`Sending image ${imagePath}`);
+					try{
+						// arrives at worker as /data/os.img.gz
+						await this.sendFile(imagePath, TARGET_PATH, 'worker');
+					}catch(e){
+						console.log(e);
+						throw new Error(`Rysnc error: ${e.message}`)
+					}
+				},
+				{
+					max_tries: 5,
+					interval: 1000 * 5,
+					throw_original: true
+				}
+			)
+		} 
+
+		// note: for the qemu case, we are assuming a shared volume between core and worker - this means that any image we want to flash must be in this volume - is it in all our cases?
+		this.logger.log(`Trying to flash ${TARGET_PATH}`);
+		await new Promise(async (resolve, reject) => {
+			let res = rp.post(
+				{ 
+					uri: `${this.url}/dut/flashFromFile`, 
+					body: {
+						filename: TARGET_PATH
+					}, 
+					timeout: 0,
+					json: true 
+				}
+			);
+
+			req.on('data', (data) => {
+				this.logger.log(data.toString());
+			});
+
+			req.finally(() => {
+				resolve();
+			});		
+		});
 	}
 
 
@@ -522,12 +510,12 @@ module.exports = class Worker {
 			);
 			// todo : replace with npm package
 			await exec(
-				`rsync -av -e "ssh ${this.workerUser}@${this.workerHost} -p ${this.workerPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q ${this.sshPrefix}balena exec -i" ${filePath} ${containerId}:${destination}`,
+				`rsync -av --partial -e "ssh ${this.workerUser}@${this.workerHost} -p ${this.workerPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q ${this.sshPrefix}balena exec -i" ${filePath} ${containerId}:${destination}`,
 			);
 		} else {
 			let ip = await this.ip(target);
 			await exec(
-				`rsync -av -e "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`,
+				`rsync -av --partial -e "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`,
 			);
 		}
 	}