CP-12693 Sign kernel modules and image during kernel build (no shim)

CP-12694 Sign ZFS modules after ZFS build (no shim)
CP-12695 Sign connstat module after build (no shim)

PR URL: https://www.github.com/delphix/linux-pkg/pull/371

Author: tonynguien

.github/scripts/verify-query-packages.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -ex
 # shellcheck disable=SC2012
+# shellcheck disable=SC2010
 
 set -o pipefail
 
@@ -31,11 +32,11 @@ test ${#fields[@]} -eq 1
 test "${fields[0]}" == 'https://github.com/delphix/zfs.git'
 
 # Expect that "list all" outputs all directory names under packages/
-diff <(ls -1 packages | sort) <(./query-packages.sh list all 2>&1 | sort)
+diff <(ls -1 packages | grep -v common.sh | sort) <(./query-packages.sh list all 2>&1 | sort)
 
 # Expect that outputing dependencies & git-url for all packages works and that the output
 # length corresponds to the number of packages.
-test "$(ls -1 packages | wc -l)" -eq \
+test "$(ls -1 packages | grep -c -v common.sh)" -eq \
 	"$(./query-packages.sh list -o name,dependencies,can-update,git-url all 2>&1 | wc -l)"
 
 # Check that all package lists under package-lists\ can be loaded and that each
@@ -76,6 +77,6 @@ test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o depend
 # Check that executing query-packages works from another directory.
 # This redoes the "list all" test from above
 cd packages
-diff <(ls -1 | sort) <(../query-packages.sh list all 2>&1 | sort)
+diff <(ls -1 | grep -v common.sh | sort) <(../query-packages.sh list all 2>&1 | sort)
 
 echo "All tests passed"

default-package-config.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright 2018, 2020 Delphix
+# Copyright 2018, 2025 Delphix
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,9 @@
 # be overriden.
 #
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/packages/common.sh"
+
 function fetch() {
 	logmust fetch_repo_from_git
 }
@@ -160,6 +163,44 @@ function kernel_build() {
 	#
 	logmust fakeroot debian/rules printenv "${debian_rules_args[@]}"
 
+	#
+	# Configure signing keys/certs before build
+	#
+	# CONFIG_MODULE_SIG_KEY is set to /var/tmp/sbkeys/signing_key.pem in
+	# resources/delphix_kernel_annotations
+	#
+	FLAVOUR=$platform
+	OBJ=debian/build/build-$FLAVOUR
+	CERTS=$OBJ/certs
+
+	# ensure the objdir + certs dir exist
+	mkdir -p "$CERTS"
+	download_keys
+
+	# provide the key the packaging expects INSIDE the objdir
+	# (symlink or copy)
+	logmust ln -sf "${SB_KEYS_DIR}/signing_key.pem" "$CERTS/signing_key.pem"
+	logmust chmod 600 "$CERTS/signing_key.pem"
+
+	# create the DER .x509 that sign-file needs from .crt)
+	logmust openssl x509 -in "${SB_KEYS_DIR}/db.crt" -outform DER -out "$CERTS/signing_key.x509"
+
+	# sanity checks
+	logmust test -s "$CERTS/signing_key.pem" || {
+		echo "missing signing_key.pem"
+		exit 1
+	}
+	logmust test -s "$CERTS/signing_key.x509" || {
+		echo "missing signing_key.x509"
+		exit 1
+	}
+	logmust openssl pkey -in "$CERTS/signing_key.pem" -noout >/dev/null || {
+		echo "key unreadable"
+		exit 1
+	}
+	SBSIGN_KEY="${SBSIGN_KEY:-$SB_KEYS_DIR/db.key}"
+	SBSIGN_CERT="${SBSIGN_CERT:-$SB_KEYS_DIR/db.crt}"
+
 	#
 	# The default value of the tool argument for mk-build-deps
 	# is the following:
@@ -203,6 +244,23 @@ function kernel_build() {
 	# one of the .debs produced
 	#
 	logmust test -f "artifacts/linux-image-${kernel_version}_"*.deb
+
+	#
+	# After the build, unpackage linux-image package and sign vmlinuz
+	#
+	linux_deb=$(find artifacts -type f -name "linux-image-${kernel_version}*.deb" | head -n1)
+	temp_dir=$(mktemp -d -p "/var/tmp/")
+	logmust fakeroot dpkg-deb -R $linux_deb "$temp_dir"
+
+	bz="$temp_dir/boot/vmlinuz-${kernel_version}"
+	logmust sbsign --key $SBSIGN_KEY --cert $SBSIGN_CERT --output "$bz.signed" "$bz"
+	logmust mv "$bz.signed" "$bz"
+	logmust sbverify --list "$bz"
+
+	# Repack the .deb"
+	update_md5sums "$temp_dir"
+	repack_deb $linux_deb $temp_dir
+	delete_keys
 }
 
 #

packages/common.sh

@@ -0,0 +1,74 @@
+#
+# Copyright 2025 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck shell=bash
+# shellcheck disable=SC2034
+# common helpers...
+
+#
+# S3 bucket containing keys and certs
+# ./db subdirectory contains the db key and various certs:
+#   .der is for signing modules like ZFS and connstat
+#   .crt is for signing vmlinuz
+#   signing_key.pem is the format expected by kernel build for signing its modules
+#
+# ./pub contains the auth files, secure boot enrollment certs.
+#
+S3_KEYS_URL="s3://secure-boot-keys-prod/temp"
+#
+# The kernel build expects the signing_key.pem in this directory, i.e.
+# CONFIG_MODULE_SIG_KEY is set to /var/tmp/sbkeys/signing_key.pem in
+# resources/delphix_kernel_annotations
+#
+SB_KEYS_DIR="/var/tmp/sbkeys"
+SBSIGN_KEY="$SB_KEYS_DIR/db.key"
+SBSIGN_DER="$SB_KEYS_DIR/db.der"
+
+function download_keys() {
+	logmust mkdir -p $SB_KEYS_DIR
+	logmust aws s3 cp --recursive "$S3_KEYS_URL/db/" $SB_KEYS_DIR
+}
+
+function delete_keys() {
+	logmust rm -r $SB_KEYS_DIR
+}
+
+#
+# Update DEBIAN/md5sum for package directory after
+# some files were updated, i.e. secure-boot signed.
+#
+function update_md5sums() {
+	pkg_dir=$1
+	echo_bold "Updating md5sums for $pkg_dir"
+
+	(
+		cd "$pkg_dir" || exit
+		: >DEBIAN/md5sums
+		# print paths relative to root of package
+		while IFS= read -r -d '' f; do
+			rel="${f#./}"
+			md5sum "$rel" >>DEBIAN/md5sums
+		done < <(find . -type f ! -path './DEBIAN/*' ! -path './etc/depmod*' -print0)
+	)
+}
+
+function repack_deb() {
+	deb_name=$1
+	deb_dir=$2
+	temp_deb=$(mktemp /tmp/deb.XXXXXX)
+
+	logmust fakeroot dpkg-deb -b "$deb_dir" "$temp_deb"
+	logmust mv "$temp_deb" "$deb_name"
+}

packages/connstat/config.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright 2018, 2020 Delphix
+# Copyright 2018, 2025 Delphix
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,9 @@
 #
 # shellcheck disable=SC2034
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../common.sh"
+
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git"
 PACKAGE_DEPENDENCIES="@linux-kernel dwarves"
 
@@ -50,4 +53,32 @@ function build() {
 
 	logmust cd "$WORKDIR/repo"
 	logmust mv ./*deb "$WORKDIR/artifacts/"
+
+	# Sign the module
+	sign_module
+}
+
+#
+# Unpack connstat module package, sign, then repack
+#
+function sign_module() {
+	echo_bold "Signing connstat module"
+	download_keys
+
+	find "$WORKDIR/artifacts" -type f -name "connstat-module-*.deb" ! -name "*-dbg*" -print0 |
+		while IFS= read -r -d '' connstat_pkg; do
+			echo_bold "Processing $connstat_pkg"
+			temp_dir=$(mktemp -d -p "/var/tmp/")
+			logmust fakeroot dpkg-deb -R "$connstat_pkg" "$temp_dir"
+
+			connstat_mod=$(find "$temp_dir" -type f -name connstat.ko)
+			logmust kmodsign sha256 "$SBSIGN_KEY" "$SBSIGN_DER" "$connstat_mod" "$connstat_mod.signed"
+			logmust mv "$connstat_mod.signed" "$connstat_mod"
+			logmust modinfo -F signer "$connstat_mod"
+
+			# Repack the .deb"
+			update_md5sums "$temp_dir"
+			repack_deb "$connstat_pkg" "$temp_dir"
+		done
+	delete_keys
 }

packages/zfs/config.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright 2019, 2020 Delphix
+# Copyright 2019, 2025 Delphix
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,9 @@
 #
 # shellcheck disable=SC2034
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../common.sh"
+
 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git"
 PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves"
 
@@ -174,6 +177,39 @@ function build() {
 	done
 	logmust cd "$WORKDIR"
 	logmust mv "all-packages/"*.deb "artifacts/"
+
+	# Sign ZFS modules in all packages
+	sign_zfs_modules
+}
+
+#
+# Unpack zfs-modules packages, sign, then repack
+#
+function sign_zfs_modules() {
+	echo_bold "Signing ZFS modules"
+	download_keys
+
+	find "$WORKDIR/artifacts" -type f -name "zfs-modules-*.deb" ! -name "*-dbg*" -print0 |
+		while IFS= read -r -d '' zfs_pkg; do
+			echo_bold "Processing $zfs_pkg"
+			temp_dir=$(mktemp -d -p "/var/tmp/")
+			logmust fakeroot dpkg-deb -R "$zfs_pkg" "$temp_dir"
+
+			zfs=$(find "$temp_dir" -type f -name zfs.ko)
+			spl=$(find "$temp_dir" -type f -name spl.ko)
+
+			logmust kmodsign sha256 "$SBSIGN_KEY" "$SBSIGN_DER" "$zfs" "$zfs.signed"
+			logmust kmodsign sha256 "$SBSIGN_KEY" "$SBSIGN_DER" "$spl" "$spl.signed"
+			logmust mv "$zfs.signed" "$zfs"
+			logmust mv "$spl.signed" "$spl"
+			logmust modinfo -F signer "$zfs"
+			logmust modinfo -F signer "$spl"
+
+			# Repack the .deb"
+			update_md5sums "$temp_dir"
+			repack_deb "$zfs_pkg" "$temp_dir"
+		done
+	delete_keys
 }
 
 function update_upstream() {

resources/delphix_kernel_annotations

@@ -2,6 +2,9 @@
 # FORMAT: 4
 # ARCH: amd64
 # FLAVOUR: amd64-aws amd64-azure amd64-generic amd64-gcp amd64-oracle
+#
+CONFIG_MODULE_SIG_KEY  		policy<{'amd64': '"/var/tmp/sbkeys/signing_key.pem"'}>
+CONFIG_MODULE_SIG_FORCE 			policy<{'amd64': 'y', 'arm64': 'y'}>
 
 #
 # Disable various "net" modules which we don't use.