Working? local test setup

3 files changed, 555 insertions, 0 deletions

diff --git a/efitest b/efitest
new file mode 100755
index 0000000..db7790b
--- /dev/null
+++ b/efitest
@@ -0,0 +1,150 @@
+#!/bin/bash
+
+MEM=2048
+SMP="-smp 1"
+ARCH=""
+DIST=""
+STORAGE=""
+OUTPUT=""
+
+usage() {
+    echo "$0"
+    echo
+    echo "Usage: $0 <-a ARCH> <-d DIST> <-s STORAGE> [ -o OUTPUT ]" 1>&2
+    echo
+    echo "Run a VM for testing Secure Boot".
+    echo
+    echo "-a ARCH      Select architecture; amd64/i386/arm64"
+    echo "             (default $ARCH)"
+    echo "-d DIST      Select Debian release to use: buster/bullseye/unstable"
+    echo "             (default $DIST)"
+    echo "-s STORAGE   Select which firmware storage image to use: MS_DEBIAN/snakeoil/SB_OFF"
+    echo "             (default: $STORAGE)"
+    echo "-o OUTPUT    Write details of the VM started to the file OUTPUT"
+    exit 1
+}
+
+validate_arg() {
+    local VAR=$1
+    local VAROPTS="$(echo $2 | tr ':' ' ')"
+
+    for OPT in $VAROPTS; do
+        if [ "${!VAR}"x = "$OPT"x ]; then
+            return
+        fi
+    done
+    # else
+    echo "$VAR ${!VAR} not supported - use one of $VAROPTS"
+    exit 1
+}
+
+while getopts ":a:d:s:o:" o; do
+    case "${o}" in
+        a)
+            ARCH=${OPTARG}
+            ;;
+        d)
+            DIST=${OPTARG}
+            ;;
+        s)
+            STORAGE=${OPTARG}
+            ;;
+        o)
+            OUTPUT=$(realpath ${OPTARG})
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+validate_arg ARCH "amd64:i386:arm64"
+validate_arg DIST "buster:bullseye:unstable"
+validate_arg STORAGE "SB_OFF:SB_OFF_debug:MS_DEBIAN:MS_DEBIAN_debug:snakeoil:snakeoil_debug"
+
+case $ARCH in
+    amd64)
+	QEMU_MACH="-M pc-i440fx-2.8,accel=kvm"
+	MACH=x86_64;;
+    i386)
+	QEMU_MACH="-machine q35,smm=on,accel=kvm"
+	MACH=i386;;
+    arm64)
+	QEMU_MACH="--enable-kvm -M virt,gic-version=host,kernel_irqchip=on"
+	MACH=aarch64;;
+esac
+
+TUPLE="$ARCH-$DIST"
+case $TUPLE in
+    amd64-buster)
+        OFFSET=1;;
+    i386-buster)
+        OFFSET=2;;
+    amd64-bullseye)
+        OFFSET=3;;
+    i386-bullseye)
+        OFFSET=4;;
+    amd64-unstable)
+        OFFSET=5;;
+    i386-unstable)
+        OFFSET=6;;
+
+    arm64-buster)
+        OFFSET=1;;
+    arm64-bullseye)
+        OFFSET=3;;
+    arm64-unstable)
+        OFFSET=5;;
+esac
+
+VNC_BASE=1
+IP_BASE=2
+VM_IP=127.16.2.$(($IP_BASE + $OFFSET))
+VNC=":"$(($VNC_BASE + $OFFSET))
+SSH_PORT=10022
+DISK=efi-hard-disk-$TUPLE.img
+PFLASH="-pflash $MACH-OVMF.fd -pflash $MACH-storage.fd.$STORAGE"
+DISPLAY="-display vnc=$VNC -daemonize -serial file:$TUPLE-serial.txt -k en-gb"
+PIDFILE=$TUPLE.pid
+
+qemu-system-$MACH $QEMU_MACH \
+    -name $TUPLE \
+    -pidfile $PIDFILE \
+    -cpu host \
+    -m $MEM \
+    $SMP \
+    $PFLASH \
+    -drive file=$DISK,format=raw,if=virtio \
+    -netdev user,id=usernet,hostfwd=tcp:${VM_IP}:${SSH_PORT}-10.0.2.15:22 \
+    -device virtio-net-pci,netdev=usernet \
+    $DISPLAY
+
+#echo $CMD
+#sleep 5
+#$CMD
+
+if [ "$OUTPUT"x != ""x ]; then
+    echo "VM_IP=$VM_IP" >> $OUTPUT
+    echo "SSH_PORT=$SSH_PORT" >> $OUTPUT
+    echo "PIDFILE=$PIDFILE" >> $OUTPUT
+    echo "VNC=$VNC" >> $OUTPUT
+    # echo "$0: Wrote details to $OUTPUT"
+else
+    echo "VM_IP=$VM_IP"
+    echo "SSH_PORT=$SSH_PORT"
+    echo "PIDFILE=$PIDFILE"
+    echo "VNC=$VNC"
+fi
+
+#    -drive format=raw,file=efi-hard-disk.img,if=none,id=mynvme -device nvme,drive=mynvme,serial=foo \
+#    -drive file=efi-hard-disk.img,format=raw \
+
+#    -device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864 \
+#     -serial tcp::4444,server \
+#    -vga qxl \
+#    -cpu SandyBridge \
+#    -no-user-config -nodefaults \
+#    -bios /usr/share/ovmf/OVMF.fd \
+
diff --git a/install_efi_binaries b/install_efi_binaries
new file mode 100755
index 0000000..61fc2ff
--- /dev/null
+++ b/install_efi_binaries
@@ -0,0 +1,103 @@
+#!/bin/bash
+#
+# Place a test binary into an image for EFI boot testing
+#
+
+# Set defaults
+ARCH=amd64
+DIST=buster
+FILE=""
+OUT=""
+
+usage() {
+    echo "$0"
+    echo
+    echo "Usage: $0 <-a ARCH> <-d DIST> -f <FILE> -o <IMAGE PATH>" 1>&2
+    echo
+    echo "Copy FILE into the ESP of the efi test image as PATH".
+    echo
+    echo "-a ARCH       Select architecture; amd64/i386/arm64"
+    echo "              (default $ARCH)"
+    echo "-d DIST       Select Debian release to use: buster/bullseye/unstable"
+    echo "              (default $DIST)"
+    echo "-f FILE       Select which file should be copied in."
+    echo "-o IMAGE PATH Select where the file should be copied to."
+    exit 1
+}
+
+validate_arg() {
+    local VAR=$1
+    local VAROPTS="$(echo $2 | tr ':' ' ')"
+
+    for OPT in $VAROPTS; do
+        if [ "${!VAR}"x = "$OPT"x ]; then
+            return
+        fi
+    done
+    # else
+    echo "$VAR ${!VAR} not supported - use one of $VAROPTS"
+    exit 1
+}
+
+while getopts ":a:d:f:o:" o; do
+    case "${o}" in
+        a)
+            ARCH=${OPTARG}
+            ;;
+        d)
+            DIST=${OPTARG}
+            ;;
+        f)
+            FILE=${OPTARG}
+            ;;
+        o)
+            OUT=${OPTARG}
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+validate_arg ARCH "amd64:i386:arm64"
+validate_arg DIST "buster:bullseye:unstable"
+
+if [ "$FILE"x = ""x ] || [ "$OUT"x = ""x ]; then
+    echo "Missing arg(s)"
+    usage
+    exit 1
+fi
+
+if [ ! -f "$FILE" ]; then
+    echo "Input file $FILE does not exist, abort!"
+    exit 1
+fi
+    
+TUPLE="$ARCH-$DIST"
+DISK=efi-hard-disk-$TUPLE.img
+
+if [ ! -f "$DISK" ]; then
+    echo "Disk image $DISK does not exist, abort!"
+    exit 1
+fi
+
+# OK, we've validated input. Now work out where the ESP is inside the
+# disk image. Parse fdisk output!
+OFFSET=$(fdisk -l $DISK | awk '
+      # Find the sector size
+      /^Units:/     { gsub("^.* = ","",$0); SECSIZE=$1}
+      # Find the start sector of the ESP, which is in column 2
+      /EFI System$/ { STARTSEC=$2 }
+      END           { print (SECSIZE * STARTSEC) }
+')
+# If the above fails, we'll get a zero offset
+if [ $OFFSET = 0 ]; then
+    echo "Disk image $DISK does not seem to include an ESP, abort!"
+    exit 1
+fi
+mcopy -v -o -i ${DISK}@@${OFFSET} "${FILE}" "::${OUT}"
+
+# And exit with the error code from mcp
+
diff --git a/shim-test b/shim-test
new file mode 100755
index 0000000..568183a
--- /dev/null
+++ b/shim-test
@@ -0,0 +1,302 @@
+#!/bin/sh
+
+# set -x
+
+# Do all the steps needed to test a set of signed shim binaries for a
+# given hash, distribution and architecture.
+
+GIT_DIR=shim@jack.einval.org:build/shim.git
+ARTIFACTS=shim@jack.einval.org:artifacts
+HOSTNAME=$(hostname --fqdn)
+KEYS="NOSIG snakeoil"
+BOOT_TIMEOUT=60
+LOG=""
+
+usage () {
+    cat <<EOF
+$0 <options> - sign a shim build
+
+options:
+
+ -a ARCH1[,ARCH2,...] - build is for the specified architecture(s)
+ -d DIST1[,DIST2,...] - build is for the specified Debian distribution(s)
+ -h HASH - the git hash of the build
+
+EOF
+}
+
+check_error () {
+    if [ $1 -ne 0 ]; then
+	echo "$0 $ARTSIN $HASH $DIST $ARCH failed with error $1: $2"
+	exit 1
+    fi
+}
+
+efi_to_arch () {
+    local ARCH=$1
+    case $ARCH in
+	amd64)
+            EFI=x64;;
+	i386)
+            EFI=ia32;;
+	arm64)
+            EFI=aa64;;
+    esac
+    echo "$EFI"
+}
+
+shutdown_vm () {
+    local VM_CONF=$1
+    local METHOD=$2
+    local EXTRA_CMDS=$3
+
+    . $PWD/$VM_CONF
+
+    if [ -r $PIDFILE ]; then
+	PID=$(cat $PIDFILE)
+    else
+	echo "$0: Can't read pidfile $PIDFILE"
+	return
+    fi
+
+    error=0
+    if [ "$METHOD" = "ssh" ]; then
+	if [ "$EXTRA_CMDS"x = "UPDATE-UPGRADE"x ]; then
+	    log "Running update/upgrade in VM"
+	    ssh ${VM_IP} -p${SSH_PORT} -lroot \
+		"apt-get update -y && apt-get -o Dpkg::Options::=--force-confnew -o Dpkg::Options::=--force-confmiss dist-upgrade -y --autoremove --purge" >> $LOGFILE
+	    error=$?
+	    log "  update/upgrade finished with error $error"
+	fi
+
+	MAX_WAIT=30 # how long to wait for clean shutdown
+	ssh ${VM_IP} -p${SSH_PORT} -lroot poweroff
+	TRY=0
+	while [ $TRY -lt $MAX_WAIT ]; do
+	    if [ -r $PIDFILE ] && (ps $PID >/dev/null); then
+		# echo "$0: kvm / qemu still running as pid $PID after $TRY seconds"
+		sleep 1
+		TRY=$(($TRY + 1))
+	    else
+		break
+	    fi
+	done	
+    fi
+
+    if [ -r $PIDFILE ] && (ps $PID >/dev/null); then
+	echo "$0: killing pid $PID"
+	kill -9 $PID
+    fi
+    rm -f $VM_CONF $PIDFILE
+    return $error
+}    
+
+# Test descriptions - include here for now, move them somewhere else
+# later. Schema follow:
+# (each test definition is colon-delimited, a missing entry will match
+# the default which is the first entry in the list)
+# 1. Test name
+# 2. Which shim binary to use (key name or NOSIG)
+# 3. Which SB keys to use in the VM image (MS_DEBIAN, snakeoil or SB_OFF)
+# 4. Expected result (BOOTSUCCESS, BOOTFAIL, BOOTTIMEOUT)
+# 5. Any other binaries to copy in, format
+#    <host file name 1>=<ESP file name 2>,<host file name 1>=<ESP file name 2>,...
+#    These will will be removed after the test is finished.
+#    NOT YET SUPPORTED!
+# 6. Is a test failure here fatal? (FAILFATAL, FAILOK)
+# 7. Any special commands to run on the system before shutdown? (NONE, UPDATE-UPGRADE)
+
+TESTS="test_0:NOSIG:SB_OFF:BOOTSUCCESS::FAILFATAL:UPDATE-UPGRADE"
+TESTS="$TESTS test_1:snakeoil:snakeoil:BOOTSUCCESS::FAILOK:NONE"
+TESTS="$TESTS test_2:snakeoil:MS_DEBIAN:BOOTFAIL::FAILOK:NONE"
+TESTS="$TESTS test_3:snakeoil:SB_OFF:BOOTSUCCESS::FAILOK:NONE"
+TESTS="$TESTS test_4:NOSIG:SB_OFF:BOOTSUCCESS::FAILOK:NONE"
+
+log () {
+    echo "$*"
+    echo "$*" >> $LOGFILE
+}
+
+run_tests () {
+    local DIST=$1
+    local ARCH=$2
+    local TESTTMP=$3
+    local EFI=$(efi_to_arch $ARCH)
+
+    local TESTS_RUN=0
+    local PASS=0
+    local FAIL=0
+    local INFRAFAIL=0
+
+    cd ~/test/$DIST-$ARCH
+    check_error $? "Failed to cd to image test dir ~/test/$DIST-$ARCH"
+
+    SERIAL=$ARCH-$DIST-serial.txt
+    VM_CONF=$ARCH-$DIST-vm.conf
+    LOGFILE=$ARCH-$DIST-test.log
+    rm -f $LOGFILE
+
+    for TEST in $TESTS; do
+	TESTNAME=$(echo $TEST | awk -F : '{print $1}')
+	KEY=$(echo $TEST | awk -F : '{print $2}')
+	IMGKEY=$(echo $TEST | awk -F : '{print $3}')
+	EXP_RESULT=$(echo $TEST | awk -F : '{print $4}')
+	EXTRAFILES=$(echo $TEST | awk -F : '{print $5}')
+	FATAL=$(echo $TEST | awk -F : '{print $6}')
+	EXTRA_CMDS=$(echo $TEST | awk -F : '{print $7}')
+	log ""
+	log "######################"
+	log "Running test $TESTNAME"
+	log " KEY:        $KEY"
+	log " IMGKEY:     $IMGKEY"
+	log " EXP_RESULT: $EXP_RESULT"
+	log " EXTRAFILES: $EXTRAFILES"
+	log " FATAL:      $FATAL"
+	log " EXTRA_CMDS: $EXTRA_CMDS"
+	log "######################"
+
+	TESTS_RUN=$(($TESTS_RUN + 1))
+
+	case $IMGKEY in
+	    SB_OFF*)
+		SB_GREP="secureboot: Secure boot (disabled|could not be determined)";;
+	    *)
+		SB_GREP="secureboot: Secure boot enabled";;
+	esac
+
+	# Copy our test binary into the ESP of the test OS image
+	install_efi_binaries -a $ARCH -d $DIST \
+			     -f "$TESTTMP/shim${EFI}.efi.signed-$KEY" \
+			     -o EFI/debian/shim${EFI}.efi
+	error=$?
+	if [ $error -ne 0 ]; then
+	    log "  install_efi_binaries failed, error $error"
+	    INFRAFAIL=$((INFRAFAIL + 1))
+	    RESULT=INFRAFAIL
+	    log "Expected result $EXP_RESULT, got $RESULT"
+	    continue
+	fi
+	rm -f $SERIAL
+	efitest -a $ARCH -d $DIST -s $IMGKEY -o $VM_CONF
+	error=$?
+	if [ $error -ne 0 ]; then
+	    log "  efitest failed, error $error"
+	    INFRAFAIL=$((INFRAFAIL + 1))
+	    RESULT=INFRAFAIL
+	    log "Expected result $EXP_RESULT, got $RESULT"
+	    continue
+	fi
+	CURRENT=$(date +%s)
+	START=$CURRENT
+	TAKEN=0
+	RESULT=""
+	while [ $TAKEN -lt $BOOT_TIMEOUT ]; do	    
+	    TAKEN=$(($CURRENT - $START))
+	    # Check we've booted to userland ok
+	    if grep -q -E "Debian GNU/Linux.*tty" $SERIAL; then
+		# *Also* check that we booted in the right state!
+		if grep -q -E "$SB_GREP" $SERIAL; then
+		    RESULT=BOOTSUCCESS
+		else
+		    SB_STATE=$(grep -E "secureboot: Secure boot" $SERIAL)
+		    log "SB state mismatch!"
+		    log "Expected to find $SB_GREP"
+		    log "Got $SB_STATE"
+		    RESULT=INFRAFAIL
+		fi
+		shutdown_vm $VM_CONF ssh $EXTRA_CMDS
+		error=$?
+		if [ $error != 0 ]; then
+		    INFRAFAIL=$((INFRAFAIL + 1))
+		    RESULT=INFRAFAIL
+		fi
+		break
+	    fi
+	    if grep -q "failed to load.*Access Denied" $SERIAL; then
+		RESULT=BOOTFAIL
+		shutdown_vm $VM_CONF kill
+		break
+	    fi
+	    sleep 1
+	    CURRENT=$(date +%s)
+	done
+	TAKEN=$(($CURRENT - $START))
+	if [ "$RESULT"x = ""x ]; then
+	    RESULT=BOOTTIMEOUT
+	    shutdown_vm $DIST-$ARCH-vm.conf kill
+	fi
+	log "Expected result $EXP_RESULT, got $RESULT after $TAKEN seconds"
+	echo "Serial log follows:" >> $LOGFILE
+	cat $SERIAL >> $LOGFILE
+	if [ "$RESULT" = "$EXP_RESULT" ]; then
+	    PASS=$(($PASS + 1))
+	else
+	    if [ "$FATAL"x != "FAILOK"x ]; then
+		log "Failure here is fatal, stopping all tests"
+		send_mail -s "FATAL TEST FAILURE"
+		break
+	    fi
+	    FAIL=$(($FAIL + 1))
+	fi
+    done
+
+    log ""
+    log "######################"
+    log "TEST RESULT SUMMARY"
+    log " PASS:      $PASS"
+    log " FAIL:      $FAIL"
+    log " INFRAFAIL: $INFRAFAIL"
+    log " TOTAL;     $TESTS_RUN"
+    log "######################"
+}
+
+while getopts ":A:a:d:h:r:" o; do
+    case "${o}" in
+        a)
+            ARCHES=${OPTARG}
+            ;;
+        d)
+            DISTS=${OPTARG}
+            ;;
+        h)
+            HASH=${OPTARG}
+            ;;
+        *)
+            echo "Unknown option ${o}"
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+[ "$HASH"x != ""x ] || check_error 1 "Need to specify hash"
+[ "$DISTS"x != ""x ] || check_error 1 "Need to specify distribution(s)"
+[ "$ARCHES"x != ""x ] || check_error 1 "Need to specify architecture(S)"
+
+cleanup () {
+    if [ "$TESTTMPS"x != ""x ]; then
+	echo "Cleaning up build dir(s) $TESTTMPS"
+	rm -rf "$TESTTMPS"
+    fi
+}
+trap "cleanup" EXIT
+
+for ARCH in $(echo "$ARCHES" | tr ',' ' '); do
+    for DIST in $(echo "$DISTS" | tr ',' ' '); do
+
+	TESTTMP=$(mktemp -d -p /dev/shm test-shim-$ARCH-$DIST-$HASH-XXXXXXXXXX)
+	check_error $? "mktemp failed"
+	TESTTMPS="$TESTTMPS $TESTTMP"
+
+	# HACK HACK not unstable!
+	rsync -av "$ARTIFACTS/sign/$HASH/$ARCH-$DIST/" "$TESTTMP/"
+
+	run_tests "$DIST" "$ARCH" "$TESTTMP"
+
+	#echo "Copy test logs to $ARTIFACTS/test-logs/$HASH/ :"
+	#rsync -av --exclude shim.git "$TESTTMP/$HASH/" "$ARTIFACTS/test-logs/$HASH/"
+    done
+done
+
+exit 0