fix: ship Pi images with MontaukOS UEFI boot entry

docs/raspberry-pi.md

@@ -118,6 +118,15 @@ the firmware to load Limine from. The card is padded to
 `RPI4_QEMU_CARD_MIB` (default 4096), and every run starts from a fresh
 copy, so every run is a first boot.
 
+The image's `RPI_EFI.fd` comes with a **MontaukOS** boot entry first in the
+firmware's boot order (`scripts/rpi4-efi-bootorder.py` writes it into the
+release's empty variable store), so the card boots on its own instead of
+falling behind the PXE/HTTP entries the firmware creates. The entry names the
+ESP by the image's MBR disk ID and partition number, so it works from the SD
+slot or a USB drive. A card whose `RPI_EFI.fd` predates it keeps its old
+boot order until it is updated with *Update the UEFI firmware*
+(`--firmware`), which also copies the image's disk ID to the card.
+
 The firmware defaults are what MontaukOS needs; leave them:
 
 - **System Table Mode: ACPI.** The kernel reads ACPI only.

scripts/mk-rpi4-sd.sh

@@ -33,6 +33,13 @@
 # the image with e.g.
 #   sudo dd if=montauk-rpi4.img of=/dev/sdX bs=4M conv=fsync
 #
+# Boot order: the firmware's RPI_EFI.fd ships with an empty variable store,
+# which scripts/rpi4-efi-bootorder.py seeds with a "MontaukOS" Boot0000 first
+# in BootOrder, naming the ESP by this image's (random) MBR disk ID and
+# partition number. The card then boots without a trip to the boot menu,
+# ahead of the PXE/HTTP entries the firmware adds on its own, from the SD
+# slot or a USB drive alike.
+#
 # Firmware settings (UEFI setup, Esc at boot) MontaukOS relies on are the
 # defaults: System Table Mode "ACPI" and "Limit RAM to 3 GB" enabled (the
 # BCM2711's PCIe can only DMA below 3 GB, see the _DMA objects in its ACPI
@@ -150,13 +157,19 @@ if [ "$ROOTFS" = ext2 ]; then
         -d "$WORK/root" "$WORK/root.ext2" "${root_mib}M"
 fi
 
+# The MBR disk ID the firmware's boot entry names the ESP by.
+DISK_ID=0
+while [ $((DISK_ID)) -eq 0 ]; do
+    DISK_ID="0x$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')"
+done
+
 rm -f "$OUTPUT"
 dd if=/dev/zero of="$OUTPUT" bs=1M count=0 seek=$((1 + esp_mib + root_mib)) status=none
 if [ "$ROOTFS" = ext2 ]; then
-    printf 'label: dos\nstart=2048, size=%dMiB, type=ef\nstart=%dMiB, type=83\n' \
-        "$esp_mib" "$((1 + esp_mib))"
+    printf 'label: dos\nlabel-id: %s\nstart=2048, size=%dMiB, type=ef\nstart=%dMiB, type=83\n' \
+        "$DISK_ID" "$esp_mib" "$((1 + esp_mib))"
 else
-    printf 'label: dos\nstart=2048, type=ef\n'
+    printf 'label: dos\nlabel-id: %s\nstart=2048, type=ef\n' "$DISK_ID"
 fi | PATH=$PATH:/usr/sbin:/sbin sfdisk --quiet "$OUTPUT"
 
 # The ESP is built as its own file: mtools sizes a filesystem at an @@offset
@@ -174,6 +187,10 @@ mformat -i "$IMG" -F -v MONTAUK ::
 for f in "$FW_DIR"/*; do
     case "$(basename "$f")" in
         Readme.md) ;;
+        RPI_EFI.fd)
+            python3 -I "$ROOT/scripts/rpi4-efi-bootorder.py" "$f" "$WORK/RPI_EFI.fd" \
+                "$DISK_ID" 1 2048 $((esp_mib * 2048))
+            mcopy -i "$IMG" "$WORK/RPI_EFI.fd" :: ;;
         config.txt)
             { cat "$f"; printf '\ncore_freq=500\ncore_freq_min=500\n'; } > "$WORK/config.txt"
             mcopy -i "$IMG" "$WORK/config.txt" :: ;;

scripts/montauk-imager.py

@@ -26,6 +26,9 @@
 #                                 if it differs, and the UEFI firmware
 #                                 RPI_EFI.fd (which holds the firmware's own
 #                                 settings) is only replaced with --firmware.
+#                                 That also copies the image's MBR disk ID to
+#                                 the card: the new RPI_EFI.fd's MontaukOS
+#                                 boot entry finds the ESP by it.
 #           root (ext2, MONTAUKROOT)
 #                                 os/, apps/, sdk/, boot/ and any other system
 #                                 directory: new and changed files written.
@@ -644,6 +647,20 @@ def expand_root(rep, name):
 
 # ==== Flash ====
 
+def copy_disk_id(rep, name, raw):
+    """Give the card the image's MBR disk ID (the firmware's boot entry names the ESP by it)."""
+    with open(raw, "rb") as f:
+        disk_id = f.read(512)[440:444]
+    fd = os.open(f"/dev/{name}", os.O_RDWR | os.O_EXCL | os.O_CLOEXEC)
+    try:
+        if os.pread(fd, 4, 440) != disk_id:
+            os.pwrite(fd, disk_id, 440)
+            os.fsync(fd)
+            rep.log(f"MBR disk ID set to 0x{struct.unpack('<I', disk_id)[0]:08x}")
+    finally:
+        os.close(fd)
+
+
 def cmd_flash(rep, args):
     img = Image(args.image)
     name = check_target(rep, args.device)
@@ -1025,6 +1042,8 @@ def update(rep, args, img, name):
             if args.expand:
                 expand_root(rep, name)
                 esp, root, _ = card_layout(rep, name)
+        if args.firmware:
+            copy_disk_id(rep, name, raw)
 
         with contextlib.ExitStack() as stack:
             rep.stage("Mounting")

scripts/rpi4-efi-bootorder.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# rpi4-efi-bootorder.py - make MontaukOS the Pi UEFI firmware's first boot option
+#
+# Usage: rpi4-efi-bootorder.py <RPI_EFI.fd> <output.fd> <disk-id> <partition> <start-lba> <sectors>
+#        (run by mk-rpi4-sd.sh)
+#
+# The firmware (github.com/pftf/RPi4, EDK2) keeps its UEFI variables in an
+# EDK2 variable store inside RPI_EFI.fd, empty in a release. Left to itself it
+# builds the boot order from whatever it enumerates (SD, USB, PXE and HTTP over
+# IPv4 and IPv6, ...), appending entries it finds later, so the card can end up
+# behind the network entries and booting it means picking it from the boot
+# menu. This writes two variables into the empty store:
+#
+#   Boot0000   "MontaukOS": HD(<partition>, MBR, <disk-id>)/\EFI\BOOT\BOOTAA64.EFI
+#   BootOrder  { 0000 }
+#
+# The device path is the short form that starts at the partition: EDK2 finds
+# it by the MBR disk ID and partition number on any block device (connecting
+# everything the first time it looks, then caching the full path), so it
+# boots the card whether it is in the SD slot or written to a USB drive. The
+# firmware appends the entries it creates after it, and never deletes it (it
+# only prunes the entries it created itself). The image's MBR must carry the
+# same disk ID; montauk-imager.py's `update --firmware` copies it to the card.
+
+import struct
+import sys
+import uuid
+
+NV_FV_GUID = uuid.UUID("fff12b8d-7696-4c8b-a985-2747075b4f50").bytes_le     # gEfiSystemNvDataFvGuid
+AUTH_STORE_GUID = uuid.UUID("aaf32c78-947b-439a-a180-2e144ec37792").bytes_le  # gEfiAuthenticatedVariableGuid
+GLOBAL_GUID = uuid.UUID("8be4df61-93ca-11d2-aa0d-00e098032b8c").bytes_le     # gEfiGlobalVariableGuid
+
+STORE_FORMATTED = 0x5A
+STORE_HEALTHY = 0xFE
+VARIABLE_DATA = 0x55AA
+VAR_ADDED = 0x3F
+NV_BS_RT = 0x7                  # NON_VOLATILE | BOOTSERVICE_ACCESS | RUNTIME_ACCESS
+LOAD_OPTION_ACTIVE = 0x1
+LOADER = "\\EFI\\BOOT\\BOOTAA64.EFI"
+
+
+def fail(msg):
+    print(f"rpi4-efi-bootorder: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def find_store(fd):
+    """Offsets of the variable store header and of its end."""
+    for fv in range(0, len(fd) - 0x48, 0x1000):
+        if fd[fv + 0x10:fv + 0x20] == NV_FV_GUID and fd[fv + 0x28:fv + 0x2C] == b"_FVH":
+            store = fv + struct.unpack_from("<H", fd, fv + 0x30)[0]
+            guid, size, fmt, state = struct.unpack_from("<16sIBB", fd, store)
+            if guid != AUTH_STORE_GUID or fmt != STORE_FORMATTED or state != STORE_HEALTHY:
+                fail("the NV variable store is not an authenticated EDK2 store")
+            return store, store + size
+    fail("no NV variable store in the firmware image")
+
+
+def variable(name, data):
+    # AUTHENTICATED_VARIABLE_HEADER (packed, 60 bytes), name, data; the next
+    # header starts 4-byte aligned, the gap left erased.
+    name = (name + "\0").encode("utf-16-le")
+    v = struct.pack("<HBBIQ16sIII16s", VARIABLE_DATA, VAR_ADDED, 0, NV_BS_RT, 0, bytes(16), 0,
+                    len(name), len(data), GLOBAL_GUID) + name + data
+    return v + b"\xff" * (-len(v) % 4)
+
+
+def load_option(desc, disk_id, part, start, sectors):
+    hd = struct.pack("<BBHIQQ16sBB", 4, 1, 42, part, start, sectors,
+                     struct.pack("<I", disk_id).ljust(16, b"\0"), 1, 1)   # MBR, MBR signature
+    path = (LOADER + "\0").encode("utf-16-le")
+    dp = hd + struct.pack("<BBH", 4, 4, 4 + len(path)) + path + struct.pack("<BBH", 0x7F, 0xFF, 4)
+    return struct.pack("<IH", LOAD_OPTION_ACTIVE, len(dp)) + (desc + "\0").encode("utf-16-le") + dp
+
+
+def main():
+    if len(sys.argv) != 7:
+        fail("usage: rpi4-efi-bootorder.py <RPI_EFI.fd> <output.fd> <disk-id> <partition> <start-lba> <sectors>")
+    src, out = sys.argv[1], sys.argv[2]
+    disk_id, part, start, sectors = (int(a, 0) for a in sys.argv[3:])
+    if not 0 < disk_id <= 0xFFFFFFFF:
+        fail(f"bad MBR disk ID {sys.argv[3]}")
+
+    fd = bytearray(open(src, "rb").read())
+    store, end = find_store(fd)
+    first = (store + 28 + 3) & ~3
+    if struct.unpack_from("<H", fd, first)[0] != 0xFFFF:
+        fail("the firmware's variable store is not empty; seed only a release RPI_EFI.fd")
+
+    blob = (variable("Boot0000", load_option("MontaukOS", disk_id, part, start, sectors))
+            + variable("BootOrder", struct.pack("<H", 0)))
+    if first + len(blob) > end:
+        fail("the variable store is too small")
+    fd[first:first + len(blob)] = blob
+    open(out, "wb").write(fd)
+
+
+if __name__ == "__main__":
+    main()