Merge pull request #21 from learningequality/rebase_upstream

Update image

Author: jredrejo

.buildkite/build.sh

@@ -44,6 +44,13 @@ jobs:
       with:
         repository: learningequality/pi-gen
         ref: ${{ inputs.ref }}
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+    - name: Download the debfile from URL and install
+      if: ${{ inputs.deb-url }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y qemu-user-static
     - name: Download the debfile from URL and install
       if: ${{ inputs.deb-url }}
       run: make get-deb deb=${{ inputs.deb-url }}

.buildkite/pipeline.yaml

@@ -4,4 +4,3 @@ dist/*
 postrun.sh
 .pc
 *-pc
-apt-cacher-ng/

.github/workflows/build_zip.yml

@@ -0,0 +1,4 @@
+include:
+  - project: serge/pi-gen
+    ref: ci
+    file: 'pi-gen.yml'

.gitignore

@@ -1,14 +1,16 @@
-FROM i386/debian:buster
-LABEL build=pigen
-ENV DEBIAN_FRONTEND noninteractive
+FROM i386/debian:bullseye
+ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get -y update && \
-    apt-get -y install \
+    apt-get -y install --no-install-recommends \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
-        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod\
+        libarchive-tools libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc \
+        binfmt-support ca-certificates fdisk gpg pigz arch-test \
     && rm -rf /var/lib/apt/lists/*
 
+COPY --from=multiarch/qemu-user-static:latest /usr/bin/qemu-arm-static /usr/bin/
+
 COPY . /pi-gen/
 
 VOLUME [ "/pi-gen/work", "/pi-gen/deploy"]

.gitlab-ci.yml

@@ -23,14 +23,16 @@ get-deb: clean-deb
 	fi
 
 rename-zip:
+# Get current date in YYYY-MM-DD format
+	$(eval DATE := $(shell date +%Y-%m-%d))
 # Unzip the file so we can rename the image file
-	unzip -o deploy/image_Kolibri-lite.zip -d deploy
+	unzip -o deploy/image_$(DATE)-Kolibri-lite.zip -d deploy
 # Clean up the original zip file
-	rm deploy/image_Kolibri-lite.zip
+	rm deploy/image_$(DATE)-Kolibri-lite.zip
 # Get the version based on the debian file name kolibri_<version>-0ubuntu1_all.deb
 	$(eval VERSION=$(shell ls ${DIST_DIR} | grep kolibri | sed 's/kolibri_\(.*\)-0ubuntu1_all.deb/\1/'))
 # Rename the image file to include the version
-	mv deploy/Kolibri-lite.img deploy/kolibri-pi-image-$(VERSION).img
+	mv deploy/$(DATE)-Kolibri-lite.img deploy/kolibri-pi-image-$(VERSION).img
 # Zip the image file back up
 	zip -j deploy/kolibri-pi-image-$(VERSION).zip deploy/kolibri-pi-image-$(VERSION).img
 # Clean up the extracted image file

Dockerfile

@@ -11,15 +11,17 @@
 5. Enjoy it!
 
 
+This image sets up the Raspberry Pi to provide wifi using the essid `kolibri` without any password.
 
 After following the above steps the Raspberry Pi will provide a wifi network named under the essid `kolibri` without any password.
 
-After connecting a device to this wifi network provided by the Raspberry, you can open the url http://10.10.10.10 in a browser will allow you enjoy all the features of a working Kolibri server.
+After connecting a device to this wifi network provided by the Raspberry, you can open the url http://10.10.10.10 in a browser will allow you enjoy all the features of a working Kolibri server. 
 
 By default the server does not have Internet access. To add content channels to Kolibri you can either connect an usb disk with content or plug and ethernet cable with Internet access.
 
 In case you want to login into the server, the user is `pi` and the password is `kolibrifly`
 
+VERY IMPORTANT: After installing the image, a ssh server is installed with a known password. CHANGE IT in case you want to connect it to Internet or be used by people who could mess it up.
 **VERY IMPORTANT NOTICE**: After installing the image, a ssh server is installed with a known password. **CHANGE IT** in case you want to connect it to Internet or be used by people who could mess it up.
 
 
@@ -41,6 +43,6 @@ Using the `raspi-config` script we'll
 
 - Add options to make the wifi encrypted
 
-
+  
 
 ## Rebuilding the Kolibri Raspberry Pi image

Makefile

@@ -1,12 +1,20 @@
-#!/bin/bash -eu
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+#!/usr/bin/env bash
+# Note: Avoid usage of arrays as MacOS users have an older version of bash (v3.x) which does not supports arrays
+set -eu
+
+DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)"
 
 BUILD_OPTS="$*"
 
-DOCKER="docker"
+# Allow user to override docker command
+DOCKER=${DOCKER:-docker}
 
-if ! ${DOCKER} ps >/dev/null 2>&1; then
-	DOCKER="sudo docker"
+# Ensure that default docker command is not set up in rootless mode
+if \
+  ! ${DOCKER} ps    >/dev/null 2>&1 || \
+    ${DOCKER} info 2>/dev/null | grep -q rootless \
+; then
+	DOCKER="sudo ${DOCKER}"
 fi
 if ! ${DOCKER} ps >/dev/null; then
 	echo "error connecting to docker:"
@@ -32,7 +40,7 @@ done
 
 # Ensure that the configuration file is an absolute path
 if test -x /usr/bin/realpath; then
-	CONFIG_FILE=$(realpath -s "$CONFIG_FILE")
+	CONFIG_FILE=$(realpath -s "$CONFIG_FILE" || realpath "$CONFIG_FILE")
 fi
 
 # Ensure that the confguration file is present
@@ -41,12 +49,13 @@ if test -z "${CONFIG_FILE}"; then
 	exit 1
 else
 	# shellcheck disable=SC1090
-	source "${CONFIG_FILE}"
+	source ${CONFIG_FILE}
 fi
 
 CONTAINER_NAME=${CONTAINER_NAME:-pigen_work}
 CONTINUE=${CONTINUE:-0}
 PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0}
+PIGEN_DOCKER_OPTS=${PIGEN_DOCKER_OPTS:-""}
 
 if [ -z "${IMG_NAME}" ]; then
 	echo "IMG_NAME not set in 'config'" 1>&2
@@ -73,31 +82,89 @@ fi
 # Modify original build-options to allow config file to be mounted in the docker container
 BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')"
 
-${DOCKER} build -t pi-gen "${DIR}"
+# Check the arch of the machine we're running on. If it's 64-bit, use a 32-bit base image instead
+case "$(uname -m)" in
+  x86_64|aarch64)
+    BASE_IMAGE=i386/debian:bullseye
+    ;;
+  *)
+    BASE_IMAGE=debian:bullseye
+    ;;
+esac
+${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
+
 if [ "${CONTAINER_EXISTS}" != "" ]; then
-	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
-	time ${DOCKER} run --rm --privileged \
-		--volume "${CONFIG_FILE}":/config:ro \
-		-e "GIT_HASH=${GIT_HASH}" \
-		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
-		pi-gen \
-		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static || true &&
-	cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
-	rsync -av work/*/build.log deploy/" &
-	wait "$!"
+  DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont"
+  DOCKER_CMDLINE_PRE="--rm"
+  DOCKER_CMDLINE_POST="--volumes-from=${CONTAINER_NAME}"
 else
-	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
-	time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
-		--volume "${CONFIG_FILE}":/config:ro \
-		-e "GIT_HASH=${GIT_HASH}" \
-		pi-gen \
-		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static || true &&
-	cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
-	rsync -av work/*/build.log deploy/" &
-	wait "$!"
+  DOCKER_CMDLINE_NAME="${CONTAINER_NAME}"
+  DOCKER_CMDLINE_PRE=""
+  DOCKER_CMDLINE_POST=""
+fi
+
+# Check if binfmt_misc is required
+binfmt_misc_required=1
+case $(uname -m) in
+  aarch64)
+    binfmt_misc_required=0
+    ;;
+  arm*)
+    binfmt_misc_required=0
+    ;;
+esac
+
+# Check if qemu-arm-static and /proc/sys/fs/binfmt_misc are present
+if [[ "${binfmt_misc_required}" == "1" ]]; then
+  if ! qemu_arm=$(which qemu-arm-static) ; then
+    echo "qemu-arm-static not found (please install qemu-user-static)"
+    exit 1
+  fi
+  if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
+    echo "binfmt_misc required but not mounted, trying to mount it..."
+    if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then
+        echo "mounting binfmt_misc failed"
+        exit 1
+    fi
+    echo "binfmt_misc mounted"
+  fi
+  if ! grep -q "^interpreter ${qemu_arm}" /proc/sys/fs/binfmt_misc/qemu-arm* ; then
+    # Register qemu-arm for binfmt_misc
+    reg="echo ':qemu-arm-rpi:M::"\
+"\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:"\
+"\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:"\
+"${qemu_arm}:F' > /proc/sys/fs/binfmt_misc/register"
+    echo "Registering qemu-arm for binfmt_misc..."
+    sudo bash -c "${reg}" 2>/dev/null || true
+  fi
 fi
+
+trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM
+time ${DOCKER} run \
+  $DOCKER_CMDLINE_PRE \
+  --name "${DOCKER_CMDLINE_NAME}" \
+  --privileged \
+  ${PIGEN_DOCKER_OPTS} \
+  --volume "${CONFIG_FILE}":/config:ro \
+  -e "GIT_HASH=${GIT_HASH}" \
+  $DOCKER_CMDLINE_POST \
+  pi-gen \
+  bash -e -o pipefail -c "
+    dpkg-reconfigure qemu-user-static || true &&
+    # binfmt_misc is sometimes not mounted with debian bullseye image
+    (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) &&
+    cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
+    rsync -av work/*/build.log deploy/
+  " &
+  wait "$!"
+
+# Ensure that deploy/ is always owned by calling user
 echo "copying results from deploy/"
-${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
+${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf -
+
+echo "copying log from container ${CONTAINER_NAME} to deploy/"
+${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log
+
 ls -lah deploy
 
 # cleanup