Chainloading does not work on mirrored EFI partitions (w/mdadm)

Support
1

Problem:
Red Hat’s official solution on redundant, mirrored EFI partitions works fine and creates two EFI partitions, both entries added to the boot list and the host is able to boot just fine (if I skip PXE boot): How to create a backup EFI partition as /boot/efi2 - Red Hat Customer Portal
I’ve gone through: UEFI booting and RAID1 « codeblog and confirmed the metadata is version 1.0 etc… (see details a bottom).

However the chainloader fails to boot from the EFI partition with this error:

image
image1131×816 107 KB

Expected outcome:
The chainloader should discover both EFI partitions and attempt to boot from them.

Foreman and Proxy versions:
satellite-6.14.4-1.el8sat.noarch / foreman-3.7.0.13-1.el8sat.noarch

Foreman and Proxy plugin versions:

Name Version
foreman-tasks 8.1.4
foreman_ansible 12.0.6
foreman_azure_rm 2.2.9
foreman_bootdisk 21.2.1
foreman_discovery 22.0.4
foreman_google 1.0.4
foreman_hooks 0.3.17
foreman_kubevirt 0.1.9
foreman_leapp 1.1.0
foreman_openscap 7.0.0
foreman_remote_execution 10.1.3
foreman_rh_cloud 8.0.51
foreman_templates 9.4.0
foreman_theme_satellite 12.0.0.9
foreman_vault 1.2.0
foreman_virt_who_configure 0.5.20
foreman_webhooks 3.2.1
katello 4.9.0.25

Distribution and version:

root@iu-satellite:~# cat /etc/redhat-release
Red Hat Enterprise Linux release 8.9 (Ootpa)
root@iu-satellite:~# uname -r
4.18.0-513.24.1.el8_9.x86_64

Other relevant data:

└─# blkid | grep efi
/dev/sdb2: UUID="1c88a982-9dda-5e55-51f6-c3b693b2cb63" UUID_SUB="718577ac-7355-5f04-5d4b-7155367b0cdb" LABEL="localhost.localdomain:efiboot" TYPE="linux_raid_member" PARTUUID="555284bb-bef3-459c-b844-bf2871f8e03a"
/dev/sdf2: UUID="1c88a982-9dda-5e55-51f6-c3b693b2cb63" UUID_SUB="f692a5ac-d796-c312-9759-c1d2c47b37e3" LABEL="localhost.localdomain:efiboot" TYPE="linux_raid_member" PARTUUID="cf13b9f4-9bcb-4631-9888-ac20db2f5610"
/dev/md126: SEC_TYPE="msdos" LABEL_FATBOOT="efiboot" LABEL="efiboot" UUID="5857-4F5A" BLOCK_SIZE="512" TYPE="vfat"

└─# mdadm  --detail /dev/md126
mdadm: Value "localhost.localdomain:boot" cannot be set as name. Reason: Not POSIX compatible. Value ignored.
mdadm: Value "localhost.localdomain:efiboot" cannot be set as name. Reason: Not POSIX compatible. Value ignored.
mdadm: Value "localhost.localdomain:os_pv" cannot be set as name. Reason: Not POSIX compatible. Value ignored.
/dev/md126:
           Version : 1.0
     Creation Time : Tue Jun  4 17:57:20 2024
        Raid Level : raid1
        Array Size : 524224 (511.94 MiB 536.81 MB)
     Used Dev Size : 524224 (511.94 MiB 536.81 MB)
      Raid Devices : 2
     Total Devices : 2
       Persistence : Superblock is persistent

     Intent Bitmap : Internal

       Update Time : Tue Jun  4 20:44:07 2024
             State : clean
    Active Devices : 2
   Working Devices : 2
    Failed Devices : 0
     Spare Devices : 0

Consistency Policy : bitmap

              Name : localhost.localdomain:efiboot
              UUID : 1c88a982:9dda5e55:51f6c3b6:93b2cb63
            Events : 48

    Number   Major   Minor   RaidDevice State
       0       8       82        0      active sync   /dev/sdf2
       1       8       18        1      active sync   /dev/sdb2

└─# file -s /dev/sdb2
/dev/sdb2: DOS/MBR boot sector, code offset 0x3c+2, OEM-ID "mkfs.fat", sectors/cluster 16, reserved sectors 16, root entries 512, Media descriptor 0xf8, sectors/FAT 256, sectors/track 4, sectors 1048448 (volumes > 32 MB), serial number 0x58574f5a, label: "efiboot    ", FAT (16 bit)

└─# file -s /dev/sdf2
/dev/sdf2: DOS/MBR boot sector, code offset 0x3c+2, OEM-ID "mkfs.fat", sectors/cluster 16, reserved sectors 16, root entries 512, Media descriptor 0xf8, sectors/FAT 256, sectors/track 4, sectors 1048448 (volumes > 32 MB), serial number 0x58574f5a, label: "efiboot    ", FAT (16 bit)

└─# efibootmgr
BootCurrent: 0011
Timeout: 3 seconds
BootOrder: 0011,0004,0005,0006,0007,0008,0009,000B,000C,000D,000E,000F,0010,0001,0000,0003
Boot0000* Red Hat Enterprise Linux      HD(2,GPT,cf13b9f4-9bcb-4631-9888-ac20db2f5610,0x200800,0x100000)/File(\EFI\redhat\shimx64.efi)
Boot0001* Red Hat Enterprise Linux      HD(2,GPT,555284bb-bef3-459c-b844-bf2871f8e03a,0x200800,0x100000)/File(\EFI\redhat\shimx64.efi)
Boot0003* UEFI: Built-in EFI Shell      VenMedia(5023b95c-db26-429b-a648-bd47664c8012)0000424f
Boot0004* UEFI: HTTP IPv4 Intel(R) Ethernet Controller X550     PciRoot(0x0)/Pci(0x1c,0x0)/Pci(0x0,0x0)/MAC(502fa8c79dca,1)/IPv4(0.0.0.00.0.0.0,0,0)/Uri()0000424f
Boot0005* UEFI: PXE IPv4 Intel(R) Ethernet Controller X550      PciRoot(0x0)/Pci(0x1c,0x0)/Pci(0x0,0x0)/MAC(502fa8c79dca,1)/IPv4(0.0.0.00.0.0.0,0,0)0000424f
Boot0006* UEFI: HTTP IPv6 Intel(R) Ethernet Controller X550     PciRoot(0x0)/Pci(0x1c,0x0)/Pci(0x0,0x0)/MAC(502fa8c79dca,1)/IPv6([::]:<->[::]:,0,0)/Uri()0000424f
Boot0007* UEFI: HTTP IPv4 Intel(R) Ethernet Controller X550     PciRoot(0x0)/Pci(0x1c,0x0)/Pci(0x0,0x1)/MAC(502fa8c79dcb,1)/IPv4(0.0.0.00.0.0.0,0,0)/Uri()0000424f
Boot0008* UEFI: PXE IPv4 Intel(R) Ethernet Controller X550      PciRoot(0x0)/Pci(0x1c,0x0)/Pci(0x0,0x1)/MAC(502fa8c79dcb,1)/IPv4(0.0.0.00.0.0.0,0,0)0000424f
Boot0009* UEFI: HTTP IPv6 Intel(R) Ethernet Controller X550     PciRoot(0x0)/Pci(0x1c,0x0)/Pci(0x0,0x1)/MAC(502fa8c79dcb,1)/IPv6([::]:<->[::]:,0,0)/Uri()0000424f
Boot000B* UEFI: HTTP IPv4 Cisco NIC c0:2c:17:2e:24:ac   PciRoot(0x3)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x1,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/MAC(c02c172e24ac,0)/IPv4(0.0.0.00.0.0.0,0,0)/Uri()0000424f
Boot000C* UEFI: PXE IPv4 Cisco NIC c0:2c:17:2e:24:ac    PciRoot(0x3)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x1,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/MAC(c02c172e24ac,0)/IPv4(0.0.0.00.0.0.0,0,0)0000424f
Boot000D* UEFI: HTTP IPv6 Cisco NIC c0:2c:17:2e:24:ac   PciRoot(0x3)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x1,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/MAC(c02c172e24ac,0)/IPv6([::]:<->[::]:,0,0)/Uri()0000424f
Boot000E* UEFI: HTTP IPv4 Cisco NIC c0:2c:17:2e:24:ad   PciRoot(0x3)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x1,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x0,0x1)/MAC(c02c172e24ad,0)/IPv4(0.0.0.00.0.0.0,0,0)/Uri()0000424f
Boot000F* UEFI: PXE IPv4 Cisco NIC c0:2c:17:2e:24:ad    PciRoot(0x3)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x1,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x0,0x1)/MAC(c02c172e24ad,0)/IPv4(0.0.0.00.0.0.0,0,0)0000424f
Boot0010* UEFI: HTTP IPv6 Cisco NIC c0:2c:17:2e:24:ad   PciRoot(0x3)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x1,0x0)/Pci(0x0,0x0)/Pci(0x0,0x0)/Pci(0x0,0x1)/MAC(c02c172e24ad,0)/IPv6([::]:<->[::]:,0,0)/Uri()0000424f
Boot0011* UEFI: Cisco vKVM-Mapped vDVD1.24      PciRoot(0x0)/Pci(0x14,0x0)/USB(5,0)/USB(2,0)/CDROM(1,0x80b,0x800)0000424f
MirroredPercentageAbove4G: 0.00
MirrorMemoryBelow4GB: false

└─# efibootmgr -v # NOTE: I removed non-relevant entries from the verbose output
BootCurrent: 0011
Timeout: 3 seconds
BootOrder: 0011,0004,0005,0006,0007,0008,0009,000B,000C,000D,000E,000F,0010,0001,0000,0003
Boot0000* Red Hat Enterprise Linux      HD(2,GPT,cf13b9f4-9bcb-4631-9888-ac20db2f5610,0x200800,0x100000)/File(\EFI\redhat\shimx64.efi)
      dp: 04 01 2a 00 02 00 00 00 00 08 20 00 00 00 00 00 00 00 10 00 00 00 00 00 f4 b9 13 cf cb 9b 31 46 98 88 ac 20 db 2f 56 10 02 02 / 04 04 34 00 5c 00 45 00 46 00 49 00 5c 00 72 00 65 00 64 00 68 00 61 00 74 00 5c 00 73 00 68 00 69 00 6d 00 78 00 36 00 34 00 2e 00 65 00 66 00 69 00 00 00 / 7f ff 04 00
Boot0001* Red Hat Enterprise Linux      HD(2,GPT,555284bb-bef3-459c-b844-bf2871f8e03a,0x200800,0x100000)/File(\EFI\redhat\shimx64.efi)
      dp: 04 01 2a 00 02 00 00 00 00 08 20 00 00 00 00 00 00 00 10 00 00 00 00 00 bb 84 52 55 f3 be 9c 45 b8 44 bf 28 71 f8 e0 3a 02 02 / 04 04 34 00 5c 00 45 00 46 00 49 00 5c 00 72 00 65 00 64 00 68 00 61 00 74 00 5c 00 73 00 68 00 69 00 6d 00 78 00 36 00 34 00 2e 00 65 00 66 00 69 00 00 00 / 7f ff 04 00

Kickstart partition config that was used:

bootloader --location=mbr --append="nofb quiet splash=quiet"

# Clear the Master Boot Record
zerombr

# Partition clearing information
clearpart --all --initlabel --disklabel=gpt

## /boot
part raid.01    --fstype="mdmember" --size=1024 --ondisk="/dev/sdk"
part raid.02    --fstype="mdmember" --size=1024 --ondisk="/dev/sdl"
raid /boot      --level=1 --device=boot --label=boot --fstype="xfs" raid.01 raid.02

## /boot/efi
part raid.11    --fstype="mdmember" --size=512 --ondisk="/dev/sdk"
part raid.12    --fstype="mdmember" --size=512 --ondisk="/dev/sdl"
raid /boot/efi  --level=1 --device=efiboot --label=efiboot --fstype="efi" raid.11 raid.12

## /
part raid.21    --fstype="mdmember" --size=1 --ondisk="/dev/sdk" --grow
part raid.22    --fstype="mdmember" --size=1 --ondisk="/dev/sdl" --grow
raid pv.01      --level=1 --device=os_pv --label=os_pv raid.21 raid.22

## Additional data disks
part pv.data.0 --size=1 --ondisk="/dev/sda" --grow
part pv.data.1 --size=1 --ondisk="/dev/sdb" --grow
part pv.data.2 --size=1 --ondisk="/dev/sdc" --grow
part pv.data.3 --size=1 --ondisk="/dev/sde" --grow
part pv.data.4 --size=1 --ondisk="/dev/sdf" --grow
part pv.data.5 --size=1 --ondisk="/dev/sdg" --grow
part pv.data.6 --size=1 --ondisk="/dev/sdh" --grow
part pv.data.7 --size=1 --ondisk="/dev/sdi" --grow
volgroup vg_data pv.data.0 pv.data.1 pv.data.2 pv.data.3 pv.data.4 pv.data.5 pv.data.6 pv.data.7
logvol /data --fstype="xfs" --size=1 --name=lv_data --vgname=vg_data --grow

# Create volume group for OS
volgroup vg.01 pv.01

# Create logical volumes for OS
logvol /                  --name=lv_root           --vgname=vg.01 --size=15360 --fstype=xfs
logvol /var               --name=lv_var            --vgname=vg.01 --size=20480 --fstype=xfs --fsoptions="nodev,nosuid"
logvol /var/log           --name=lv_var_log        --vgname=vg.01 --size=20480 --fstype=xfs --fsoptions="nodev,noexec,nosuid"
logvol /tmp               --name=lv_tmp            --vgname=vg.01 --size=20480 --fstype=xfs --fsoptions="nodev,noexec,nosuid"
logvol /var/log/audit     --name=lv_var_log_audit  --vgname=vg.01 --size=4096  --fstype=xfs --fsoptions="nodev,noexec,nosuid"
2 
root@iu-satellite:/var/log# cat /var/lib/tftpboot/grub2/grub.cfg-01-c0-2c-17-2e-24-ac
set default=local
set timeout=20
echo Default PXE local template entry is set to 'local'

insmod part_gpt
insmod fat
insmod chain

echo "VMWare hosts with QuickBoot feature enabled may not find the local ESP"
echo "partition due to not initializing all the EFI devices. To workaround, upgrade"
echo "to the latest grub2 (*) and uncomment "connectefi scsi" statement in the"
echo "grub2_chainload template."
echo
echo "Virtual or physical hosts using Software RAID for the ESP partition may try"
echo "booting on the Software RAID, which will fail. To workaround, upgrade to the"
echo "latest grub2 (*) and add "--efidisk-only" argument to the "search" command in"
echo "the grub2_chainload template."
echo
echo "(*) grub2-efi-x64-2.02-122.el8 (upstream doesn't have the patches yet)"
echo
#connectefi scsi

menuentry 'Chainload Grub2 EFI from ESP' --id local_chain_hd0 {
  echo "Chainloading Grub2 EFI from ESP, enabled devices for booting:"
  ls
  echo "Trying /EFI/fedora/shim.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/fedora/shim.efi
  if [ -f ($chroot)/EFI/fedora/shim.efi ]; then
    chainloader ($chroot)/EFI/fedora/shim.efi
    echo "Found /EFI/fedora/shim.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/fedora/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/fedora/grubx64.efi
  if [ -f ($chroot)/EFI/fedora/grubx64.efi ]; then
    chainloader ($chroot)/EFI/fedora/grubx64.efi
    echo "Found /EFI/fedora/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/redhat/shim.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/redhat/shim.efi
  if [ -f ($chroot)/EFI/redhat/shim.efi ]; then
    chainloader ($chroot)/EFI/redhat/shim.efi
    echo "Found /EFI/redhat/shim.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/redhat/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/redhat/grubx64.efi
  if [ -f ($chroot)/EFI/redhat/grubx64.efi ]; then
    chainloader ($chroot)/EFI/redhat/grubx64.efi
    echo "Found /EFI/redhat/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/centos/shim.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/centos/shim.efi
  if [ -f ($chroot)/EFI/centos/shim.efi ]; then
    chainloader ($chroot)/EFI/centos/shim.efi
    echo "Found /EFI/centos/shim.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/centos/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/centos/grubx64.efi
  if [ -f ($chroot)/EFI/centos/grubx64.efi ]; then
    chainloader ($chroot)/EFI/centos/grubx64.efi
    echo "Found /EFI/centos/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/rocky/shim.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/rocky/shim.efi
  if [ -f ($chroot)/EFI/rocky/shim.efi ]; then
    chainloader ($chroot)/EFI/rocky/shim.efi
    echo "Found /EFI/rocky/shim.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/rocky/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/rocky/grubx64.efi
  if [ -f ($chroot)/EFI/rocky/grubx64.efi ]; then
    chainloader ($chroot)/EFI/rocky/grubx64.efi
    echo "Found /EFI/rocky/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/debian/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/debian/grubx64.efi
  if [ -f ($chroot)/EFI/debian/grubx64.efi ]; then
    chainloader ($chroot)/EFI/debian/grubx64.efi
    echo "Found /EFI/debian/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/ubuntu/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/ubuntu/grubx64.efi
  if [ -f ($chroot)/EFI/ubuntu/grubx64.efi ]; then
    chainloader ($chroot)/EFI/ubuntu/grubx64.efi
    echo "Found /EFI/ubuntu/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/sles/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/sles/grubx64.efi
  if [ -f ($chroot)/EFI/sles/grubx64.efi ]; then
    chainloader ($chroot)/EFI/sles/grubx64.efi
    echo "Found /EFI/sles/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/opensuse/grubx64.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/opensuse/grubx64.efi
  if [ -f ($chroot)/EFI/opensuse/grubx64.efi ]; then
    chainloader ($chroot)/EFI/opensuse/grubx64.efi
    echo "Found /EFI/opensuse/grubx64.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Trying /EFI/Microsoft/boot/bootmgfw.efi "
  unset chroot
  # add --efidisk-only when using Software RAID
  search --file --no-floppy --set=chroot /EFI/Microsoft/boot/bootmgfw.efi
  if [ -f ($chroot)/EFI/Microsoft/boot/bootmgfw.efi ]; then
    chainloader ($chroot)/EFI/Microsoft/boot/bootmgfw.efi
    echo "Found /EFI/Microsoft/boot/bootmgfw.efi at $chroot, attempting to chainboot it..."
    sleep 2
    boot
  fi
  echo "Partition with known EFI file not found, you may want to drop to grub shell"
  echo "and investigate available files updating 'pxegrub2_chainload' template and"
  echo "the list of known filepaths for probing. Available devices are:"
  echo
  ls
  echo
  echo "If you cannot see the HDD, make sure the drive is marked as bootable in EFI and"
  echo "not hidden. Boot order must be the following:"
  echo "1) NETWORK"
  echo "2) HDD"
  echo
  echo "The system will poweroff in 2 minutes or press ESC to poweroff immediately."
  sleep -i 120
  halt
}

menuentry 'Chainload into BIOS bootloader on first disk' --id local_chain_legacy_hd0 {
  set root=(hd0,0)
  chainloader +1
  boot
}

menuentry 'Chainload into BIOS bootloader on second disk' --id local_chain_legacy_hd1 {
  set root=(hd1,0)
  chainloader +1
  boot
}

common="rootflags=loop root=live:/fdi.iso rootfstype=auto ro rd.live.image acpi=force rd.luks=0 rd.md=0 rd.dm=0 rd.lvm=0 rd.bootif=0 rd.neednet=0 nokaslr nomodeset proxy.url=https://iu-satellite.domain.com proxy.type=foreman BOOTIF=01-$net_default_mac"

if [ ${grub_platform} == "pc" ]; then
  menuentry 'Foreman Discovery Image' --id discovery {
    linux boot/fdi-image/vmlinuz0 ${common}
    initrd boot/fdi-image/initrd0.img
  }
else
  menuentry 'Foreman Discovery Image EFI' --id discovery {
    linuxefi boot/fdi-image/vmlinuz0 ${common}
    initrdefi boot/fdi-image/initrd0.img
  }
fi

root@iu-satellite:/var/log#