Download BalenaOS images from URL

client/lib/config-validator.js

@@ -4,6 +4,16 @@
 let report = { verdict: false, errors: [] };
 let counter = 1;
 
+function isUrl(filePath) {
+	let pattern = new RegExp('^(https?:\\/\\/)?' + // protocol
+		'(\\S+\\:\\S+@)?' + // optional user:pass authentication
+		'((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|' + // domain name
+		'((\\d{1,3}\\.){3}\\d{1,3}))' + // OR ip (v4) address
+		'(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*' + // port and path
+		'(\\?[;&a-z\\d%_.~+=-]*)?' + // query string
+		'(\\#[-a-z\\d_]*)?$', 'i'); // fragment locator
+	return !!pattern.test(filePath);
+}
 async function testConfig(configs) {
   for (const config of configs) {
     const balenaCloud = new BalenaCloudInteractor(config.config.balenaApiUrl);
@@ -50,6 +60,8 @@
     try {
       if (config.image === false) {
         console.log(`False flag provided, tests would be downloading ${config.deviceType} ${config.config.downloadVersion} image later ✅\n`)
+      } else if (isUrl(config.image)) {
+        console.log(`Image URL provided ✅\n`)
       } else {
         fs.accessSync(config.image, fs.F_OK);
         console.log(`Testing using OS image: #${config.image} ✅\n`)

client/lib/index.js

@@ -26,11 +26,22 @@
 	return buf[0] === 0x1f && buf[1] === 0x8b && buf[2] === 0x08;
 }
 
+function isUrl(filePath) {
+	let pattern = new RegExp('^(https?:\\/\\/)?' + // protocol
+		'(\\S+\\:\\S+@)?' + // optional user:pass authentication
+		'((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|' + // domain name
+		'((\\d{1,3}\\.){3}\\d{1,3}))' + // OR ip (v4) address
+		'(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*' + // port and path
+		'(\\?[;&a-z\\d%_.~+=-]*)?' + // query string
+		'(\\#[-a-z\\d_]*)?$', 'i'); // fragment locator
+	return !!pattern.test(filePath);
+}
+
 async function isZip(filepath) {
 	const buf = Buffer.alloc(4);
 
 	await fs.read(await fs.open(filepath, 'r'), buf, 0, 4, 0)
-	
+
 	// Referencing ZIP file signatures https://www.garykessler.net/library/file_sigs.html
 	return buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x03 || buf[2] === 0x05 || buf[2] === 0x07) && (buf[3] === 0x04 || buf[3] === 0x06 || buf[3] === 0x08);
 }
@@ -309,13 +320,13 @@
 
 			let heartbeatFailCounter = 0
 			const heartbeat = setInterval(async () => {
-				try{
+				try {
 					await rp.get(`${this.uri}/heartbeat`);
 					heartbeatFailCounter = 0
-				} catch{
+				} catch {
 					console.log(`Failed to ping worker hearbeat: ${heartbeatFailCounter} failures in a row`)
 					heartbeatFailCounter += 1
-					if(heartbeatFailCounter === 10){
+					if (heartbeatFailCounter === 10) {
 						throw new Error(`Lost communication with worker! `)
 					}
 				}
@@ -344,7 +355,7 @@
 								case 'image':
 									// [Hack] Upload a fake image if image is false
 									// Remove when https://github.com/balena-os/leviathan/issues/567 is resolved
-									if (suiteConfig.image === false) {
+									if (suiteConfig.image === false || isUrl(suiteConfig.image)) {
 										// Had to create fake image in home directory otherwise
 										// facing a permission issue since client root is read-only
 										const fakeImagePath = '/home/test-balena.img.gz';

core/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
 	unzip \
 	util-linux \
 	wget \
-	vim \
+	curl \
 	build-essential \
 	make \
 	python && \

core/lib/common/downloadFromUrl.js

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2024 balena
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const { spawn } = require("child_process");
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const retry = require('bluebird-retry');
+const config = require('../config');
+const { join } = require('path');
+
+// Function for generating SHA-256 checksum
+function generateChecksum(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash('MD5'); 
+    const stream = fs.createReadStream(filePath);
+
+    stream.on('data', (data) => hash.update(data));
+    stream.on('end', () => resolve(hash.digest('hex')));
+    stream.on('error', (error) => reject(error));
+  });
+}
+
+module.exports = {
+
+  /**
+   * Download a file with `curl`.
+   * 
+   * @param {string} downloadUrl - The URL to download the file from.
+   * @param {string[]} otherArgs - Optional arguments to pass to curl.
+   *
+   * @category helper
+   * @remark https://www.jenkins.io/doc/book/system-administration/authenticating-scripted-clients/
+   */
+  downloadFromUrl: async function (downloadUrl, otherArgs = []) {
+    const downloadFileName = new URL(downloadUrl).pathname.split('/').pop()
+    const downloadHost = new URL(downloadUrl).hostname
+    const downloadPath = join(
+      config.leviathan.downloads,
+      `${downloadFileName}`,
+    );
+
+    // Prepare curl command with necessary options
+    const args = [
+      // "-vvv", // Debugging options
+      '--silent', "--show-error", // No Progress Bar
+      '-C', '-', '--fail', // Fail options
+      '--connect-timeout', '1060', '--keepalive-time', '1060', // Timeout options
+      '-H', 'accept-encoding: gzip, deflate', // Headers
+      '-O', '--output-dir', config.leviathan.downloads, // Output options
+      downloadUrl, // Auth options
+      ...otherArgs // Custom options
+    ]
+
+    return retry(() => {
+        return new Promise((resolve, reject) => {
+          const curl = spawn('curl', args);
+          console.log(`Download started from ${downloadHost}... `);
+
+          curl.stdout.on('data', (data) => {
+            console.log(`${data}`);
+          });
+
+          curl.stderr.on('data', (data) => {
+            // To get any logs disable silent option
+            console.log(`${data}`); // Display progress data from curl, even errors
+          });
+
+          // when the spawn child process exits, check if there were any errors
+          curl.on('exit', function (code, signal) {
+            if (code != 0) {
+              reject(`received code ${code} and signal ${signal}`)
+            }
+          });
+
+          curl.on('close', async (code, signal) => {
+            if (code === 0) {
+              console.log(`File downloaded to ${downloadPath}. File size: ${fs.statSync(downloadPath).size / (1024 * 1024)} MB`);  // Size in Megabytes
+              console.log(`Downloaded file checksum: ${await generateChecksum(downloadPath)}`);
+              resolve(downloadPath);
+            } else {
+              reject(`received code ${code} and signal ${signal}`)
+            }
+          }),
+          {
+            max_tries: 3,
+          }
+        })
+      })
+  }
+}

suites/config.js

@@ -13,7 +13,7 @@ module.exports = [{
 			port: process.env.BALENACLOUD_SSH_PORT,
 		}
   },
-  image: false,
+  image: `https://api.balena-cloud.com/download?deviceType=${process.env.DEVICE_TYPE}`,
   debug: {
     unstable: ["Kill the device under test"],
   },

suites/e2e/suite.js

@@ -26,6 +26,17 @@
 	);
 };
 
+function isUrl(filePath) {
+	let pattern = new RegExp('^(https?:\\/\\/)?' + // protocol
+		'(\\S+\\:\\S+@)?' + // optional user:pass authentication
+		'((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|' + // domain name
+		'((\\d{1,3}\\.){3}\\d{1,3}))' + // OR ip (v4) address
+		'(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*' + // port and path
+		'(\\?[;&a-z\\d%_.~+=-]*)?' + // query string
+		'(\\#[-a-z\\d_]*)?$', 'i'); // fragment locator
+	return !!pattern.test(filePath);
+}
+
 const enableSerialConsole = async (imagePath) => {
 	const bootConfig = await imagefs.interact(imagePath, 1, async (_fs) => {
 		return util
@@ -64,6 +75,7 @@
 		// The suite contex is an object that is shared across all tests. Setting something into the context makes it accessible by every test
 		this.suite.context.set({
 			utils: this.require("common/utils"),
+			downloadFromUrl: this.require("common/downloadFromUrl"),
 			sshKeyPath: join(homedir(), "id"),
 			sshKeyLabel: this.suite.options.id,
 			sdk: new Balena(this.suite.options.balena.apiUrl, this.getLogger(), this.suite.options.config.sshConfig),
@@ -98,18 +110,26 @@
 
 		const keys = await this.utils.createSSHKey(this.sshKeyPath);
 
+		async function imagePathFinder(that) {
+			if (that.suite.options.image === false) {
+				return await that.context
+				.get()
+				.sdk.fetchOS(that.suite.options.balenaOS.download.version, that.suite.deviceType.slug)
+			} else if (isUrl(that.suite.options.image)) {
+				console.log(`Downloading image from ${that.suite.options.image}...`)
+				return await that.downloadFromUrl.downloadFromUrl(that.suite.options.image)
+			} else {
+				return undefined
+			}
+		}
+
 		// Create an instance of the balenaOS object, containing information such as device type, and config.json options
 		this.suite.context.set({
 			os: new BalenaOS(
 				{
 					deviceType: this.suite.deviceType.slug,
 					network: this.suite.options.balenaOS.network,
-					image:
-						this.suite.options.image === false
-							? `${await this.context
-									.get()
-									.sdk.fetchOS(this.suite.options.balenaOS.download.version, this.suite.deviceType.slug)}`
-							: undefined,
+					image: await imagePathFinder(this),
 					configJson: {
 						uuid: this.suite.options.balenaOS.config.uuid,
 						os: {

suites/e2e/tests/flash/index.js

@@ -26,7 +26,7 @@ module.exports = {
 				} catch (err) {
 					throw new Error(`Flashing failed with error: ${err}`);
 				}
-				test.true(true, `${this.os.image.path} should be flashed properly`);
+				test.ok(true, `${this.os.image.path} should be flashed properly`);
 			},
 		},
 	],