make-raw

#!/bin/sh
# shellcheck shell=dash
#
# This script creates a raw disk image partitioned with GPT partition table
# and set up for UEFI boot sequence with a GRUB UEFI payload and a default
# grub.cfg attempting to chainload GRUB from one of the actuall rootfs 
# partitions. This means that at the very minimim the output of this script
# will produce a disk that looks like:
#
# +----------------------------+
# |  UEFI partition            |
# |   w/ GRUB + grub.cfg       |
# +----------------------------+
# |  rootfs partition w/GRUB   |
# +----------------------------+
# 
# In addition to producing this minimalistic (but fully functional!) layout,
# this script is also capable of initializing a few additional partitions
# that are required for live upgrade and configuration:
#   * 2nd rootfs partition
#   * /config partition
#   * /persist partition
#
# The script CLI UX is really not user friendly for now, since it is expected
# to be called mostly from other scripts (and also linuxkit VMs). So no, there's
# no printing of help for this one and all the arguments are required! Here they are:
#   <img> [part1...] 
# 
# <img>       file name of the raw disk image (we expect it to be pre-created and 
#             sized correctly OR be an actual physical device)
# [part1...]  list of partitions to be created: efi imga imgb conf persist
#             Can be ommitted. Default is: efi imga imgb conf persist
# 
# On stdin, this scripts expects to recieve a tarball full of partition images
# that will be used to pre-populated actual partitions it creates. This tarball 
# is going to be recieved from stdin and extracted into /parts folder. This means, 
# of course, that if you don't supply a tarball stream to the script you can just
# pre-populate /parts with the same images:
#   * rootfs*.img for rootfs partition
#   * config.tar for config partition
#
set -e
[ -n "$DEBUG" ] && set -x

IMGFILE=$1
shift
PARTS=${*:-"efi imga imgb conf persist"}

# This is the only partition type that PARTITION_TYPE_USR_X86_64
# grub-core/commands/gptprio.c code will pay attention to
PARTITION_TYPE_USR_X86_64=5dfbf5f4-2848-4bac-aa5e-0d9a20b745a6

# EFI partition size in bytes 
EFI_PART_SIZE=$((36 * 1024 * 1024))
# rootfs partition size in bytes
ROOTFS_PART_SIZE=$(( 300 * 1024 * 1024 ))
# conf partition size in bytes
CONF_PART_SIZE=$((1024 * 1024))
# installer inventory parition size in bytes
WIN_INVENTORY_PART_SIZE=$((10240 * 1024))
# installer system parition size in bytes
INSTALLER_SYS_PART_SIZE=$(( 300 * 1024 * 1024 ))
# boot partition size
BOOT_PART_SIZE=$(( 20 * 1024 * 1024 ))
# sector where the first partition starts on a blank disk
FIRST_PART_SEC=2048

sgdisk() {
  # filter out annoying messages we can't get rid of:
  #   https://github.com/kini/gptfdisk/blob/master/diskio-unix.cc#L153
  local OUT
  local RET
  OUT=$(/usr/bin/sgdisk "$@" 2>&1)
  RET=$?
  echo "$OUT" | grep -Ev "$(echo '^Disk device is
when determining sector size! Setting sector size to 512
Warning: The kernel is still using the old partition table
The new table will be used at the next reboot or after you
run partprobe.*or kpartx
The operation has completed successfully.' | tr '\012' '|')$^" || :
  return $RET
}

cp_with_backup() {
  local MD5_SUM=$(md5sum "$2" | cut -f1 -d\  )
  [ -f "$2".$MD5_SUM ] || cp "$2" "$2".$MD5_SUM
  [ $? -eq 0 ] && cp "$1" "$2"
}

grow_part() {
  # (x+1024)/1024*1024 rounds up to multiple of 1024KB, or 2048
  # sectors some firmwares get confused if the partitions are not
  # aligned on 2048 blocks we will round up to the nearest multiple of
  # 2048 blocks since each block is 512 bytes, we want the size to be
  # a multiple of 2048 blocks * 512 bytes = 1048576 bytes = 1024KB
  # The output is in sectors though, hence the final * 2
  local IMAGE_SIZE_KB=$(( ( ( ($2 + 1024-1) / 1024 ) + 1024-1) / 1024 * 1024))
  echo $(( $1 + 2 * $IMAGE_SIZE_KB - 1))
}

dir2vfat() {
  # <img name> dir2vfat <dir> <image size> [label]
  local IMG=`mktemp -u -p /tmp/data`
  local LABEL=${3:-EVE}
  local FORCE_FAT32="-F32"
  # FAT32 can only reside on disks larger than 33Mb
  [ "$2" -lt 33792 ] && FORCE_FAT32="-v"
  (rm -rf /tmp/data
   mkdir /tmp/data
   mkfs.vfat "$FORCE_FAT32" -v -n "$LABEL" -C "$IMG" "$2"
   mcopy -i $IMG -s $1/* ::/ ) >&2
  echo $IMG
}

mkefifs() {
  rm -rf /efifs
  mkdir /efifs
  cp -Lr "$EFI_DIR" /efifs/EFI
}

do_system_vfat_part() {
  eval "local SEC_START=\$$1"
  local SEC_END="$(grow_part "$SEC_START" "$2")"
  local NUM_PART=$(( PART_OFFSET + 1 ))

  # Create a partition
  sgdisk --new "$NUM_PART:$SEC_START:$SEC_END" --typecode="$NUM_PART:ef00" --change-name="$NUM_PART":'EFI System' \
         --attributes "$NUM_PART:set:2" "$IMGFILE"

  #   ...copy EFI fs to EFI partition
  dd if="$(dir2vfat /efifs $(( (SEC_END - SEC_START) / 2)))" of="$IMGFILE" bs=512 conv=notrunc seek="$SEC_START"

  eval "$1=$((SEC_END + 1))"
}

do_efi() {
  mkefifs
  sed -e 's#@PATH_TO_GRUB@#'"$(cd /efifs; echo EFI/BOOT/BOOT*EFI)"'#' < /grub.cfg.in > /efifs/EFI/BOOT/grub.cfg
  do_system_vfat_part "$1" "$EFI_PART_SIZE"
}

do_boot() {
  mkefifs
  cp /trampoline.grub.cfg /efifs/EFI/BOOT/grub.cfg
  cp -r /parts/boot/* /efifs/
  do_system_vfat_part "$1" "$BOOT_PART_SIZE"
}

do_installer() {
  mkefifs
  cp "$INITRD_IMG" "$ROOTFS_IMG" /UsbInvocationScript.txt /efifs
  do_system_vfat_part "$1" "$INSTALLER_SYS_PART_SIZE"
}

do_rootfs() {
    eval SEC_START=\$$1
    local SEC_END=`grow_part $SEC_START $ROOTFS_PART_SIZE`
    LABEL=$2
    IMG=$3

    case $2 in
      IMGA) NUM_PART=$(( $PART_OFFSET + 2 ))
            EXTRA_ATTR="--attributes=$NUM_PART:set:56 --attributes=$NUM_PART:set:49"
            ;;
      IMGB) NUM_PART=$(( $PART_OFFSET + 3 ))
            ;;
    esac

    # Calculate partition size and add a partition
    sgdisk --new "$NUM_PART:$SEC_START:$SEC_END" \
           --typecode="$NUM_PART:$PARTITION_TYPE_USR_X86_64" \
           --change-name="$NUM_PART:$LABEL" $EXTRA_ATTR "$IMGFILE"

    # Copy rootfs to image A
    dd if=$IMG of=$IMGFILE bs=512 conv=notrunc seek=$SEC_START

    eval $1=$(( $SEC_END + 1))
}

do_imga() {
    do_rootfs $1 IMGA $ROOTFS_IMG
}

do_imgb() {
    # for now we are not initializing IMGB - hence passing /dev/null
    do_rootfs $1 IMGB /dev/null
}

do_vfat() {
    eval local SEC_START=\$$1
    local SEC_END=`grow_part $SEC_START $CONF_PART_SIZE`
    local NUM_PART=$(( $PART_OFFSET + 4 ))
    local PART_TYPE=$2

    sgdisk --new $NUM_PART:$SEC_START:$SEC_END \
           --typecode=$NUM_PART:$PART_TYPE \
	   --change-name=$NUM_PART:'CONFIG' $IMGFILE

    dd if=$CONF_FILE of=$IMGFILE bs=512 conv=notrunc seek=$SEC_START

    eval $1=$(( $SEC_END + 1))
}

do_conf() {
    do_vfat $1 13307e62-cd9c-4920-8f9b-91b45828b798 
}

do_conf_win() {
    do_vfat $1 EBD0A0A2-B9E5-4433-87C0-68B6B72699C7
}

do_inventory_win() {
    eval local SEC_START="\$$1"
    # shellcheck disable=SC2155
    local SEC_END=$(grow_part "$SEC_START" "$WIN_INVENTORY_PART_SIZE")
    local NUM_PART=$(( PART_OFFSET + 5 ))
    local PART_TYPE=EBD0A0A2-B9E5-4433-87C0-68B6B72699C7

    sgdisk --new "$NUM_PART:$SEC_START:$SEC_END" \
           --typecode="$NUM_PART:$PART_TYPE" \
           --change-name="$NUM_PART:INVENTORY" "$IMGFILE"

    # shellcheck disable=SC2046
    dd if=$(dir2vfat $(mktemp -d) $(( (SEC_END - SEC_START) / 2)) INVENTORY) of="$IMGFILE" bs=512 conv=notrunc seek="$SEC_START"

    eval "$1=$(( SEC_END + 1))"
}

do_persist() {
    eval SEC_START=\$$1
    # Persistent Purgeable Partition.  It is set at partition
    # number 9 to reserve the first 8 partitions to system types.
    local NUM_PART=$(( $PART_OFFSET + 9 ))
    # P3 takes all space available
    local SEC_END=0

    sgdisk --new $NUM_PART:$SEC_START:$SEC_END \
           --typecode=$NUM_PART:5f24425a-2dfa-11e8-a270-7b663faccc2c \
           --change-name=$NUM_PART:'P3' $IMGFILE

    [ -e $PERSIST_FILE ] && dd if=$PERSIST_FILE of=$IMGFILE bs=512 conv=notrunc seek=$SEC_START
    
    eval $1=0
}

do_usb_conf() {
    eval local SEC_START="\$$1"
    # shellcheck disable=SC2155
    local SEC_END=$(sgdisk -E "$IMGFILE")
    local NUM_PART=$(( PART_OFFSET + 1 ))
    local PART_TYPE=EBD0A0A2-B9E5-4433-87C0-68B6B72699C7
    local FAT_SIZE=$(( SEC_END - SEC_START ))

    sgdisk --new "$NUM_PART:$SEC_START:$SEC_END" \
           --typecode="$NUM_PART:$PART_TYPE" \
           --change-name="$NUM_PART:DevicePortConfig" "$IMGFILE"

    mformat -i "${IMGFILE}@@$(( SEC_START * 512 ))" -h $(( FAT_SIZE / 65535 + 1 )) -t 1 -s 65535 -l EVEDPC ::
    mcopy -i "${IMGFILE}@@$(( SEC_START * 512 ))" /parts/* ::/

    eval "$1=$(( SEC_END + 1))"
}

# This function deploys GRUB stage1 into MBR (sector 0) and GRUB stage2 into the gap betwee
# where GPT ends and first partition begins (sector 34 - sector FIRST_PART_SEC). It relies
# on the knowlege of where GRUB implemenation stores the value for the 1st and 2nd sectors
# of stage2 (see the seek offsets for dd below). Whenever GRUB version changes there's a chance
# these offsets may change and will have to be adjusted accordingly as per:
#   * grub-core/boot/i386/pc/boot.S kernel_sector
#   * grub-core/boot/i386/pc/diskboot.S blocklist_default_start
# NOTE: theoretically one can patch grub-install to do this, but it requires much more hoop jumping
deploy_legacy_grub() {
    # put GRUB stage1 loader into the MBR (don't overwrite MBR partition table -- hence size 446 not 512)
    dd if=/usr/lib/grub/i386-pc/boot.img of="$IMGFILE" bs=446 count=1 conv=noerror,sync,notrunc
    # embed GRUB stage2 into the gap between the end of GPT (sector 34) and start of the first partition (FIRST_PART_SEC)
    dd if=/efifs/EFI/BOOT/BOOT.pc of="$IMGFILE" bs=512 seek=34 conv=noerror,sync,notrunc
    # update locations where stage1 and stage2 store the value of the first (34 == \042)...
    printf '\042' | dd of="$IMGFILE" bs=1 seek=92 count=1 conv=noerror,sync,notrunc
    # ...and 2nd (35 == \043) sector of where stage2 is located on disk
    printf '\043' | dd of="$IMGFILE" bs=1 seek=$((34 * 512 + 500)) count=1 conv=noerror,sync,notrunc
}

adjust_protective_mbr() {
    # Since sgdisk by default puts protective partition first, we need to swap the
    # order to make sone legacy BIOS implementations happy. Strictly speaking, this
    # goes against good recommendations of how to build a protective MBR for the GPT
    # but it doesn't seem to cause any troubles and it helps with compabitlity.
    PROT_PART=$(busybox hexdump -s 446 -n 16 -v -e '"" 16/1 "\\%03o"' "$IMGFILE")
    NULL_PART="\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"
    # we need to replace 1st byte with \200 (bootable) and 4th byte with \014 (vfat filesystem type)
    BOOT_PART=$(busybox hexdump -s $(( 446 + 17 )) -n 3 -v -e '"\\200" 3/1 "\\%03o" "\\014"' "$IMGFILE")
    BOOT_PART="$BOOT_PART"$(busybox hexdump -s $(( 446 + 21 )) -n 11 -v -e '"" 11/1 "\\%03o"' "$IMGFILE")
    # sadly, busybox doesn't support xxd -r -- so make do with printf that interprets \ooo escape sequences
    #shellcheck disable=SC2059
    printf "${BOOT_PART}${PROT_PART}${NULL_PART}${NULL_PART}" | dd of="$IMGFILE" bs=1 seek=446 conv=noerror,sync,notrunc
}



#
# Extract partitions from stdin or look them up in /bits
#
if [ ! -d /parts ]; then
   mkdir /parts
   (cd /parts ; bsdtar xzf -)
fi

# content of initrd installer image
INITRD_IMG=/parts/initrd.img
# content of rootfs partition
ROOTFS_IMG=/parts/rootfs.img
# content of conf partition 
CONF_FILE=/parts/config.img
# content of persist partition
PERSIST_FILE=/parts/persist.img
# EFI boot directory
EFI_DIR=/parts/EFI

# offset in the GTP partition table from which we can start our numbering
PART_OFFSET=0
# starting sector for our portion of the disk
CUR_SEC="$FIRST_PART_SEC"

# Lets see if GPT partition exists and it is one of the
# kinds we recognize
case "$(sgdisk -p $IMGFILE 2>/dev/null | sed -ne '/^Number/,$s/^.* //p' | tr '\012' ' ')" in
  "Name vrl vrl_backup mcuimage fastboot nvme boot reserved cache"*)
      echo "Found Android GPT partition table on $IMGFILE"
      for p in $(sgdisk -p $IMGFILE 2>/dev/null | sed -e '1,/cache$/d' | awk '{print $1;}') ; do
        sgdisk -d $p $IMGFILE
      done 
      PART_OFFSET=10
      CUR_SEC=$(( ( $(sgdisk -p $IMGFILE 2>/dev/null | tail -1 | awk '{print $3;}') / 2048 + 1 ) * 2048 ))
      EMBED_BOOT_START=$(sgdisk -i 6 $IMGFILE 2>/dev/null | awk '/First sector:/{ print $3; }')
      EMBED_BOOT_SIZE=$(sgdisk -i 6 $IMGFILE 2>/dev/null | awk '/Partition size:/{ print $3; }')
      ;;
  "Name System IMGA IMGB CONFIG P3"*)
      echo "Found EVE GPT partition table on $IMGFILE"
      # apparently sgdisk -Z doesn't clear MBR and keeps complaining
      dd if=/dev/zero of="$IMGFILE" bs=512 count=1 conv=notrunc
      sgdisk -Z --clear $IMGFILE 2>/dev/null || :
      ;;
   *) echo "Unknown (or unrecongnizable) GTP partition table on $IMGFILE"
      # apparently sgdisk -Z doesn't clear MBR and keeps complaining
      dd if=/dev/zero of="$IMGFILE" bs=512 count=1 conv=notrunc
      sgdisk -Z --clear $IMGFILE 2>/dev/null || :
      ;;
esac

for p in $PARTS ; do
  eval do_$p CUR_SEC
done

# Create a hybrid MBR to allow booting on legacy BIOS PC systems and ARM boards that
# look for bootloaders in the first entry of the MBR
sgdisk -h$(( PART_OFFSET + 1 )) "$IMGFILE"

# if we happen to be building an x86 image - deploy legacy GRUB into the GPT gap
if [ -e /efifs/EFI/BOOT/BOOT.pc ]; then
   deploy_legacy_grub
fi

# Update embedded boot partition with our own bootloader - this only happens if
# we noticed a recongnizable GPT structure and we are ADDING ourselves to it,
# as opposed to replacing the entire GPT with our own structure (currently this
# only happens for HiKey but the approach of using partitions to store firmware
# blobs on flash is fairly common on ARM so we expect to see others as well)
# On the other hand...
GRUB_IMG="$(echo /efifs/EFI/BOOT/BOOT*.EFI)"
if [ ${EMBED_BOOT_START:-0} -gt 0 -a ${EMBED_BOOT_SIZE:-0} -gt 0 -a -f "$GRUB_IMG" ] ; then
  if mount $IMGFILE /mnt -o loop,offset=$(( EMBED_BOOT_START * 512  )),sizelimit=$(( EMBED_BOOT_SIZE * 512 )) ; then
     FASTBOOT=$(cd /mnt/EFI/BOOT/ ; ls | grep -i '^fastboot.efi$')
     if $(set ${FASTBOOT:-. .} ; test $# -eq 1) ; then
        cp_with_backup "$GRUB_IMG" "/mnt/EFI/BOOT/$FASTBOOT"
        (echo 'set root=hd0,gpt11' ; echo 'configfile /efi/boot/grub.cfg') > /tmp/grub.cfg
        cp_with_backup /tmp/grub.cfg /mnt/EFI/BOOT/grub.cfg
     fi
  fi
  umount /mnt || :
else
  # ...if we're NOT adding ourselves to an existing GPT - assume we own protective MBR
  # as well and can adjust it accordingly to maximize our chances of booting on something
  # like Raspbery Pi (which is pretty strict about how the first entry in the MBR partition
  # table needs to look like)
  adjust_protective_mbr
fi

# Validate the health of our creation
sgdisk -v $IMGFILE