Slow logical volume detection

Question

I am experiencing some issues with test server lately. It was installed few days ago and worked OK until yesterday, when it did not wake up after reboot. Bootloader did not find root device. Error:

Gave up waiting for root device.
ALERT! /dev/mapper/vg_mru-lv_mru_root does not exists. dropping to shell!

I booted in recovery shell, mounted devices, chrooted into system and everything went OK, so I just increased rootdelay to 90s and rebooted. So it worked for another day. But now I am getting same error, followed by:

udevd[112]: worker [119] unexpectedly returned with status 0x0100
udevd[112]: worker[119] failed while handling '/devices/virtual/block/md0'
udevd[112]: worker [120] unexpectedly returned with status 0x0100
udevd[112]: worker[120] failed while handling '/devices/virtual/block/md1'

If i wait for about 5 minutes and then write exit int intramfs shell, boot continue as usual and computer becomes fully functional.

In messages, there is few dozen of those lines:

Sep 30 21:58:48 mru-server kernel: [ 1038.291536] lost page write due to I/O error on dm-15

and

Sep 29 10:02:16 mru-server kernel: [    0.845246] ACPI Warning: Incorrect checksum in table [OEMB] - A4, should be 9F (20090903/tbutils-314)

Computer running ubuntu 10.04, 2 1TD disks, mirrored in raid1, two raid devices md0 and md1, each holds one LVM volume group. Both disks have passed short s.m.a.r.t. test. Memory also passed one memtest86+ pass. And it was tested for 24h 2 days ago.

Debian bug #583917 looks somehow similar, but Ubuntu has older version of mdadm.

Another thing i suspect is lvm. When i do cat /proc/modules in intramfs shell, there are dozen of raid modules, but no lvm module.

I can provide you additional data if you need.

grub.cfg

#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by /usr/sbin/grub-mkconfig using templates
# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
if [ -s $prefix/grubenv ]; then
  load_env
fi
set default="0"
if [ ${prev_saved_entry} ]; then
  set saved_entry=${prev_saved_entry}
  save_env saved_entry
  set prev_saved_entry=
  save_env prev_saved_entry
  set boot_once=true
fi

function savedefault {
  if [ -z ${boot_once} ]; then
    saved_entry=${chosen}
    save_env saved_entry
  fi
}

function recordfail {
  set recordfail=1
  if [ -n ${have_grubenv} ]; then if [ -z ${boot_once} ]; then save_env recordfail; fi; fi
}
insmod raid
insmod mdraid
insmod lvm
insmod ext2
set root='(vg_mru-lv_mru_usr)'
search --no-floppy --fs-uuid --set 48111d09-5c3d-48e4-be9d-0ab141c8cb4a
if loadfont /share/grub/unicode.pf2 ; then
  set gfxmode=640x480
  insmod gfxterm
  insmod vbe
  if terminal_output gfxterm ; then true ; else
    # For backward compatibility with versions of terminal.mod that don't
    # understand terminal_output
    terminal gfxterm
  fi
fi
insmod raid
insmod mdraid
insmod lvm
insmod ext2
set root='(vg_mru-lv_mru_boot)'
search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
set locale_dir=($root)/grub/locale
set lang=en
insmod gettext
if [ ${recordfail} = 1 ]; then
  set timeout=-1
else
  set timeout=10
fi
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/05_debian_theme ###
set menu_color_normal=white/black
set menu_color_highlight=black/light-gray
### END /etc/grub.d/05_debian_theme ###

### BEGIN /etc/grub.d/10_linux ###
menuentry 'Ubuntu, with Linux 2.6.32-25-server' --class ubuntu --class gnu-linux --class gnu --class os {
    recordfail
    insmod raid
    insmod mdraid
    insmod lvm
    insmod ext2
    set root='(vg_mru-lv_mru_boot)'
    search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
    linux   /vmlinuz-2.6.32-25-server root=/dev/mapper/vg_mru-lv_mru_root ro   rootdelay=90
    initrd  /initrd.img-2.6.32-25-server
}
menuentry 'Ubuntu, with Linux 2.6.32-25-server (recovery mode)' --class ubuntu --class gnu-linux --class gnu --class os {
    recordfail
    insmod raid
    insmod mdraid
    insmod lvm
    insmod ext2
    set root='(vg_mru-lv_mru_boot)'
    search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
    echo    'Loading Linux 2.6.32-25-server ...'
    linux   /vmlinuz-2.6.32-25-server root=/dev/mapper/vg_mru-lv_mru_root ro single 
    echo    'Loading initial ramdisk ...'
    initrd  /initrd.img-2.6.32-25-server
}
menuentry 'Ubuntu, with Linux 2.6.32-24-server' --class ubuntu --class gnu-linux --class gnu --class os {
    recordfail
    insmod raid
    insmod mdraid
    insmod lvm
    insmod ext2
    set root='(vg_mru-lv_mru_boot)'
    search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
    linux   /vmlinuz-2.6.32-24-server root=/dev/mapper/vg_mru-lv_mru_root ro   rootdelay=90
    initrd  /initrd.img-2.6.32-24-server
}
menuentry 'Ubuntu, with Linux 2.6.32-24-server (recovery mode)' --class ubuntu --class gnu-linux --class gnu --class os {
    recordfail
    insmod raid
    insmod mdraid
    insmod lvm
    insmod ext2
    set root='(vg_mru-lv_mru_boot)'
    search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
    echo    'Loading Linux 2.6.32-24-server ...'
    linux   /vmlinuz-2.6.32-24-server root=/dev/mapper/vg_mru-lv_mru_root ro single 
    echo    'Loading initial ramdisk ...'
    initrd  /initrd.img-2.6.32-24-server
}
### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_memtest86+ ###
menuentry "Memory test (memtest86+)" {
    insmod raid
    insmod mdraid
    insmod lvm
    insmod ext2
    set root='(vg_mru-lv_mru_boot)'
    search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
    linux16 /memtest86+.bin
}
menuentry "Memory test (memtest86+, serial console 115200)" {
    insmod raid
    insmod mdraid
    insmod lvm
    insmod ext2
    set root='(vg_mru-lv_mru_boot)'
    search --no-floppy --fs-uuid --set 4523fbba-a68d-413a-9606-c68094438879
    linux16 /memtest86+.bin console=ttyS0,115200n8
}
### END /etc/grub.d/20_memtest86+ ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries.  Simply type the
# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

Thanks for answers!

Regards, Andraz

score 1 · Answer 1 · answered Nov 11 '11 at 04:26

I ran into this recently upgrading a machine from Ubuntu Januty to Ubuntu Karmic. I, too, had 2 volume groups including one with around 20 logical volumes and a second one with root and some snapshots.

My issue was adding just one more LVM snapshot pushed the initialization time of the LVM subsystem past the time the kernel would wait for the root file system to be come available.

The solution was to add a "rootdelay=180" parameter to the kernel line in /boot/grub/menu.lst (this capability is one of the reasons I'm still using grub legacy).

This does have the effect of lengthening boot times but on a long running server I don't find this a significant problem.

score 0 · Answer 2 · answered Oct 01 '10 at 12:56

0

Is your disk S.M.A.R.T capable ? If yes, could you have a look it ? It seems like a disk failure to me. (Or maybe RAM failure)

answered Oct 01 '10 at 12:56

Nikolaidis Fotis

1,994
11
13

Disk is smart capable, i have run short tests and both disks passed. Maybe i forgot to add into question, i am running Ubuntu server 10.04, 2 1TB disks in raid1 and LVM setup over raid. I found debian bug #583917 quite relavant, but ubuntu is using another version of mdadm. – overlord_tm Oct 01 '10 at 13:28
Ok. Did you check your RAMs ? – Nikolaidis Fotis Oct 01 '10 at 13:36
Could you send you grub configuration ? Do you have raid module loaded ? – Nikolaidis Fotis Oct 01 '10 at 13:38
Nikolaidis, i have updated question and added grub config. In short. Both disks pass smrt test, memory passes memtest, raid module is loaded, but I think there should be lvm module also, but cant see it in /proc/modules – overlord_tm Oct 01 '10 at 14:35
Did you fix it ? – Nikolaidis Fotis Oct 05 '10 at 08:31

score 0 · Answer 3 · edited Nov 11 '11 at 05:57

0

OK, i think i solved it.

I have two volume groups, vg_mru for host system, and vg_virt for virtual systems. I have been recently testing some LVM snapshot functionality, including trying to overwrite snapshots over their capacity. I do not have exact cause yet, but removing all test logical volumes on vg_virt solved it.

edited Nov 11 '11 at 05:57

mdpc

11,698
28
51
65

answered Oct 01 '10 at 15:17

overlord_tm

133
1
6

I don't know ... maybe it's the way that grub.cfg is produced. One problem that i had is that i tried to write some code to grub.cfg by hand and it was just ignored !!! I had to use grub-mkconfig to auto-produce it. The strange is that both the auto-produced code and mine were exactly the same ! – Nikolaidis Fotis Oct 01 '10 at 16:02

score 0 · Answer 4 · answered Oct 14 '11 at 17:50

0

I believe that LVM has to do some kind of GC when snapshots are involved. When you removed the test LV on vg_virt, were there many snapshots left?

answered Oct 14 '11 at 17:50

mcr

233
1
10

Slow logical volume detection

4 Answers4