menuentry ‘Amazon Linux (4.14.128-112.105.amzn2.x86_64) 2’ –class centos rhel fedora –class gnu-linux –class gnu –class os –unrestricted $menuentry_id_option ‘gnulinux-simple-a1e1011e-e38f-408e-878b-fed395b47ad6’ {

        load_video

        insmod gzio

        insmod part_gpt

        insmod xfs

        if [ x$feature_platform_search_hint = xy ]; then

          search –no-floppy –fs-uuid –set=root  a1e1011e-e38f-408e-878b-fed395b47ad6

        else

          search –no-floppy –fs-uuid –set=root a1e1011e-e38f-408e-878b-fed395b47ad6

        fi

        linux16 /boot/vmlinuz-4.14.128-112.105.amzn2.x86_64 root=UUID=a1e1011e-e38f-408e-878b-fed395b47ad6 ro  console=tty0 console=ttyS0,115200n8 net.ifnames=0 biosdevname=0 nvme_core.io_timeout=4294967295 rd.emergency=poweroff rd.shell=0 LANG=en_US.UTF-8 KEYTABLE=us

        initrd16 /boot/initramfs-4.14.128-112.105.amzn2.x86_64.img

}

-As you can see, the default kernel being used is the following:

/boot/vmlinuz-4.14.128-112.105.amzn2.x86_64

-Which is using the following initramfs image:

initrd16 /boot/initramfs-4.14.128-112.105.amzn2.x86_64.img

-You can reproduce the error by removing the image from the /boot directory:

# cd /boot/

# ls

config-4.14.123-111.109.amzn2.x86_64  grub2                                        initrd-plymouth.img                       System.map-4.14.123-111.109.amzn2.x86_64  vmlinuz-4.14.128-112.105.amzn2.x86_64

config-4.14.128-112.105.amzn2.x86_64  initramfs-4.14.123-111.109.amzn2.x86_64.img  symvers-4.14.123-111.109.amzn2.x86_64.gz  System.map-4.14.128-112.105.amzn2.x86_64

efi                                   initramfs-4.14.128-112.105.amzn2.x86_64.img  symvers-4.14.128-112.105.amzn2.x86_64.gz  vmlinuz-4.14.123-111.109.amzn2.x86_64

# mv initramfs-4.14.128-112.105.amzn2.x86_64.img /home/ec2-user/

-As you can see above, I moved the file to the /home/ec2-user directory. After rebooting the instance, the instance will fail and if you check the console logs, you will see the kernel panic:

Kernel panic - not syncing: VFS: Unable to mount root fs on unknown-block(0,0)

-You can easily resolve the issue by moving the file back to the /boot directory. However, lets say the file is gone which may have accidentally been deleted or lost. How would we recover access to the instance then.

-Well, there is a secondary kernel you can use which uses a different initramfs file as seen here:

menuentry ‘Amazon Linux 2’ –class centos rhel fedora –class gnu-linux –class gnu –class os –unrestricted $menuentry_id_option ‘gnulinux-simple-a1e1011e-e38f-408e-878b-fed395b47ad6’ {

        load_video

        insmod gzio

        insmod part_gpt

        insmod xfs

        if [ x$feature_platform_search_hint = xy ]; then

          search –no-floppy –fs-uuid –set=root  a1e1011e-e38f-408e-878b-fed395b47ad6

        else

          search –no-floppy –fs-uuid –set=root a1e1011e-e38f-408e-878b-fed395b47ad6

        fi

        linux16 /boot/vmlinuz-4.14.123-111.109.amzn2.x86_64 root=UUID=a1e1011e-e38f-408e-878b-fed395b47ad6 ro  console=tty0 console=ttyS0,115200n8 net.ifnames=0 biosdevname=0 nvme_core.io_timeout=4294967295 rd.emergency=poweroff rd.shell=0

        initrd16 /boot/initramfs-4.14.123-111.109.amzn2.x86_64.img

}

-As you can see, the secondary kernel uses a different initramfs image:

initrd16 /boot/initramfs-4.14.123-111.109.amzn2.x86_64.img

-Before, switching to the secondary kernel, what happens when we replace the initramfs image with the secondary initramfs image?

-Lets see….

-So I detached the volume from the instance and attached to it a recovery instance as /dev/xvdh1.

[ec2-user@imraan ~]$ lsblk

NAME    MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT

xvda    202:0    0  100G  0 disk

└─xvda1 202:1    0  100G  0 part /

xvdh    202:112  0    8G  0 disk

└─xvdh1 202:113  0    8G  0 part

xvdf    202:80   0  110G  0 disk

└─xvdf1 202:81   0  110G  0 part /mnt

xvdg    202:96   0   40G  0 disk

└─xvdg1 202:97   0   40G  0 part /bitnami

-Next, to gain access to the file we needed to mount the volume to a mount point I created called /home/test

[ec2-user@imraan ~]$ sudo mount /dev/xvdh1 /home/test/

-Next, to make things easier, I switched to root:

 [ec2-user@imraan ~]$ sudo su

 [root@imraan grub2]# vi /home/test/boot/grub2/grub.cfg

-Next, I edited the grub.cfg file and replaced the following initramfs file > initrd16 /boot/initramfs-4.14.128-112.105.amzn2.x86_64.img with > initrd16 /boot/initramfs-4.14.123-111.109.amzn2.x86_64.img

-After detaching the volume and attaching it back to the source instance, the instance failed to boot:

[ec2-user@imraan ~]$ ssh -i germany.pem ec2-user@3.123.22.238

^C

-Even though we just moved the initramfs file to another folder, lets assume its gone. You would have two options here:

-Create a new one using Dracut

-Use the secondary kernel

I will show you how to do both here:

First lets rebuild initramfs using the Dracut command.

After mounting the volume, we will need to bind the following directories:

[root@imraan boot]# sudo mount -o bind /dev/ /home/test/dev

[root@imraan boot]# sudo mount -o bind /dev/shm /home/test/dev/shm

[root@imraan boot]# sudo mount -o bind /proc /home/test/proc

[root@imraan boot]# sudo mount -o bind /sys /home/test/sys

Next we will isolate our mount point using chroot.

 [root@imraan boot]# chroot /home/test/

Rebuilding initramfs is quite simple. All you will need to do it run the Dracut command as seen here:

[root@imraan /]# dracut -f

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

find: `/lib/modules/4.14.101-75.76.amzn1.x86_64/’: No such file or directory

intel-06-4f-01: model ‘GenuineIntel 06-4f-01’, path ‘ intel-ucode/06-4f-01’, kvers ‘ 4.14.42’

intel-06-4f-01: blacklist ”

intel: model ”, path ‘ intel-ucode/*’, kvers ”

intel: blacklist ”

Next, switch to the boot folder to verify the file was rebuild.

[root@imraan /]# cd /boot/

[root@imraan boot]# ls

config-4.14.123-86.109.amzn1.x86_64  initramfs-4.14.101-75.76.amzn1.x86_64.img   System.map-4.14.123-86.109.amzn1.x86_64

efi                                  initramfs-4.14.123-86.109.amzn1.x86_64.img  vmlinuz-4.14.123-86.109.amzn1.x86_64

grub                                 symvers-4.14.123-86.109.amzn1.x86_64.gz

And that’s its, detach the volume and attach it back to the instance and the instance should boot fine.

The second option as mentioned is using a secondary kernel:

We will use the same process as above by binding the following directories:

[root@imraan boot]# sudo mount -o bind /dev/ /home/test/dev

[root@imraan boot]# sudo mount -o bind /dev/shm /home/test/dev/shm

[root@imraan boot]# sudo mount -o bind /proc /home/test/proc

[root@imraan boot]# sudo mount -o bind /sys /home/test/sys

Next we will isolate our mount point using chroot.

[root@imraan boot]# chroot /home/test/

GRUB1 (Legacy GRUB) for Red Hat 6 and Amazon Linux 1

Use the sed command to replace the corrupt kernel with the stable kernel in the /boot/grub/grub.conf file.

sudo sed -i ‘/^default/ s/0/1/’ /boot/grub/grub.conf

GRUB2 for RHEL 7.5 and Amazon Linux 2

1.    Replace the corrupt GRUB_DEFAULT=0 default menu entry with the stable GRUB_DEFAULT-saved value in the /etc/default/grubfile.

sed -i ‘s/GRUB_DEFAULT=0/GRUB_DEFAULT=saved/g’ /etc/default/grub

2.    Update grub to regenerate the /boot/grub2/grub.cfg file.

sudo grub2-mkconfig -o /boot/grub2/grub.cfg

3.    Run the grub2-set-default command so that the stable kernel loads at the next reboot. In this example grub2-set-default is set to 1 in position 0.

sudo grub2-set-default 1

Next, detach the volume and that’s it.

Leave a Reply

Your email address will not be published. Required fields are marked *