8:08 p.m.
Hi, I upgraded an oVirt installation a month ago to the latest 3.6.7.
Before it was 3.6.0 if I remember correctly.
Everything went fine so far for a month or so.
A couple of days ago the the default gateway got rebooted and the
physical server hosting the HE decided to shut down the vm because it
could not ping the gateway.
The other host restarted the hevm but it now has *no nic*.
As a workaround I attached a virtio nic via virsh but every time the vm
gets restarted the nic get lost
After a bit of troubleshooting and digging this is what I found:
This is the /var/run/ovirt-hosted-engine-ha/vm.conf which, as far as I
understand, gets extracted from the HE storage domain
emulatedMachine=pc
vmId=497f5e4a-0c76-441a-b72e-724d7092d07e
smp=2
memSize=6144
spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
vmName=HostedEngine
display=vnc
devices={index:0,iface:virtio,format:raw,bootOrder:1,address:{slot:0x06,bus:0x00,domain:0x0000,type:pci,function:0x0},volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-
4f4f-bfad-0277c37f2808,readonly:false,domainID:2c3585cc-b7bc-4881-85b3-aa6514991a26,deviceId:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,poolID:00000000-0000-0000-0000-000000000000,device:disk,
shared:exclusive,propagateErrors:off,type:disk}
devices={index:2,iface:ide,shared:false,readonly:true,deviceId:8c3179ac-b322-4f5c-9449-c52e3665e0ae,address:{controller:0,target:0,unit:0,bus:1,type:drive},device:cdrom,path:,type:disk}
devices={device:cirrus,alias:video0,type:video,deviceId:a99468b6-02d4-4a77-8f94-e5df806030f6,address:{slot:0x02,bus:0x00,domain:0x0000,type:pci,function:0x0}}
devices={device:virtio-serial,type:controller,deviceId:b7580676-19fb-462f-a61e-677b65ad920a,address:{slot:0x03,bus:0x00,domain:0x0000,type:pci,function:0x0}}
devices={device:usb,type:controller,deviceId:c63092b3-7bd8-4b54-bcd3-51f34dce478a,address:{slot:0x01,bus:0x00,domain:0x0000,type:pci,function:0x2}}
devices={device:ide,type:controller,deviceId:c77c2c01-6ccc-404b-b8d6-5a7f0631a52f,address:{slot:0x01,bus:0x00,domain:0x0000,type:pci,function:0x1}}
As you can see there is no nic, and there is no nic in the qemu-kvm
command-line:
qemu 23290 1 14 00:23 ? 01:44:26 /usr/libexec/qemu-kvm
-name HostedEngine -S -machine pc-i440fx-rhel7.2.0,accel=kvm,usb=off
-cpu qemu64,-svm -m 6144 -realtime mlock=off -s
mp 2,sockets=2,cores=1,threads=1 -uuid
497f5e4a-0c76-441a-b72e-724d7092d07e -smbios
type=1,manufacturer=oVirt,product=oVirt
Node,version=7-2.1511.el7.centos.2.10,serial=4C4C4544-004B-571
0-8044-B9C04F5A3732,uuid=497f5e4a-0c76-441a-b72e-724d7092d07e
-no-user-config -nodefaults -chardev
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-HostedEngine/monitor.sock,serve
r,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
base=2016-11-30T23:23:26,driftfix=slew -global
kvm-pit.lost_tick_policy=discard -no-hpet -no-reboot -boot strict=on -device
piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device
virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.0,addr=0x3
-drive file=/var/run/vdsm/storage/2c3585cc-b7bc-4881-85b3-aa6514
991a26/d65b82e2-2ad1-4f4f-bfad-0277c37f2808/bb3218ba-cbe9-4cd0-b50b-931deae992f7,if=none,id=drive-virtio-disk0,format=raw,serial=d65b82e2-2ad1-4f4f-bfad-0277c37f2808,cache=none,werror=st
op,rerror=stop,aio=native -device
virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-drive if=none,id=drive-ide0-1-0,readonly=on,format=raw
-device ide-cd,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
-chardev
socket,id=charchannel0,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.com.redhat.rhevm
.vdsm,server,nowait -device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.rhevm.vdsm
-chardev socket,id=charchannel1,path=/var/lib/libvirt/qem
u/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.qemu.guest_agent.0,server,nowait
-device
virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=org.qemu.guest
_agent.0 -chardev
socket,id=charchannel2,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.ovirt.hosted-engine-setup.0,server,nowait
-device virtserialport,bus
=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel2,name=org.ovirt.hosted-engine-setup.0
-vnc 0:0,password -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -msg
timestamp=on
I extracted the vm.conf from the storage domain and the nic is there:
mId=497f5e4a-0c76-441a-b72e-724d7092d07e
memSize=6144
display=vnc
devices={index:2,iface:ide,address:{ controller:0, target:0,unit:0,
bus:1,
type:drive},specParams:{},readonly:true,deviceId:857b98b3-cf43-4c2d-8061-e7f105234a65,path:,device:cdrom,shared
:false,type:disk}
devices={index:0,iface:virtio,format:raw,poolID:00000000-0000-0000-0000-000000000000,volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,specParams
:{},readonly:false,domainID:2c3585cc-b7bc-4881-85b3-aa6514991a26,optional:false,deviceId:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,address:{bus:0x00,
slot:0x06, domain:0x0000, type:pci, funct
ion:0x0},device:disk,shared:exclusive,propagateErrors:off,type:disk,bootOrder:1}
devices={device:scsi,model:virtio-scsi,type:controller}
devices={nicModel:pv,macAddr:00:16:3e:7d:d8:27,linkActive:true,network:ovirtmgmt,filter:vdsm-no-mac-spoofing,specParams:{},deviceId:5be8a089-9f51-46dc-a8bd-28422985aa35,address:{bus:0x00
, slot:0x03, domain:0x0000, type:pci,
function:0x0},device:bridge,type:interface}
devices={device:console,specParams:{},type:console,deviceId:1644f556-a4ff-4c93-8945-5aa165de2a85,alias:console0}
vmName=HostedEngine
spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
smp=2
cpuType=SandyBridge
emulatedMachine=pc
The local vm.conf gets continuosly overwritten but for some reason the
nic line gets lost in the process.