Hi,
Yesterday someone powered off our storage, and all my 3 hosts lose
their disks. After 2 days of recovering i managed to bring back
everything, except engine VM, which is online but not visible to
itself.
I did new deployment of VM, restored backup and started engine setup.
After manual database updates, my all VMs and hosts are OK now, but
engine. I have engine VM with different VM id running than in
database.
I've tried this with no luck.
engine=# update vm_static set vm_guid =
'75072b32-6f93-4c38-8f18-825004072c1a' where vm_guid =(select
vm_guid from vm_static where vm_name = 'HostedEngine');
ERROR: update or delete on table "vm_static" violates foreign key
constraint "fk_disk_vm_element_vm_static" on table "disk_vm_element"
DETAIL: Key (vm_guid)=(d81ccb53-2594-49db-b69a-04c73b504c59) is still
referenced from table "disk_vm_element".
Right now i've deployed engine on all 3 hosts but see this picture:
[root@virt3 ~]# hosted-engine --vm-status
!! Cluster is in GLOBAL MAINTENANCE mode !!
[root@virt3 ~]# vdsClient -s 0 list
75072b32-6f93-4c38-8f18-825004072c1a
Status = Up
statusTime = 4397337690
kvmEnable = true
emulatedMachine = pc
afterMigrationStatus =
pid = 5280
devices = [{'device': 'console', 'specParams': {}, 'type':
'console', 'deviceId': '2b6b0e87-c86a-4144-ad39-40d5bfe25df1',
'alias': 'console0'}, {'device': 'memballoon', 'specParams': {'model':
'none'}, 'type': 'balloon', 'target': 16777216, 'alias': 'balloon0'},
{'specParams': {'source': 'random'}, 'alias': 'rng0', 'address':
{'slot': '0x07', 'bus': '0x00', 'domain': '0x0000', 'type': 'pci',
'function': '0x0'}, 'device': 'virtio', 'model': 'virtio', 'type':
'rng'}, {'device': 'unix', 'alias': 'channel0', 'type': 'channel',
'addr
ess': {'bus': '0', 'controller': '0', 'type': 'virtio-serial', 'port':
'1'}}, {'device': 'unix', 'alias': 'channel1', 'type': 'channel',
'address': {'bus': '0', 'controller': '0', 'type': 'virtio-serial',
'port': '2'}}, {'device': 'unix', 'alias': 'channel2', 'type': 'ch
annel', 'address': {'bus': '0', 'controller': '0', 'type':
'virtio-serial', 'port': '3'}}, {'device': 'scsi', 'alias': 'scsi0',
'model': 'virtio-scsi', 'type': 'controller', 'address': {'slot':
'0x04', 'bus': '0x00', 'domain': '0x0000', 'type': 'pci', 'function':
'0x0'}}
, {'device': 'usb', 'alias': 'usb', 'type': 'controller', 'address':
{'slot': '0x01', 'bus': '0x00', 'domain': '0x0000', 'type': 'pci',
'function': '0x2'}}, {'device': 'ide', 'alias': 'ide', 'type':
'controller', 'address': {'slot': '0x01', 'bus': '0x00', 'domain':
'0x00
00', 'type': 'pci', 'function': '0x1'}}, {'device': 'virtio-serial',
'alias': 'virtio-serial0', 'type': 'controller', 'address': {'slot':
'0x05', 'bus': '0x00', 'domain': '0x0000', 'type': 'pci', 'function':
'0x0'}}, {'device': 'vga', 'alias': 'video0', 'type': 'video',
'address': {'slot': '0x02', 'bus': '0x00', 'domain': '0x0000', 'type':
'pci', 'function': '0x0'}}, {'device': 'vnc', 'type': 'graphics',
'port': '5900'}, {'nicModel': 'pv', 'macAddr': '00:16:3e:01:29:95',
'linkActive': True, 'network': 'ovirtmgmt', 'alias': 'net0', 'spec
Params': {}, 'deviceId': 'd348a068-063b-4a40-9119-a3d34f6c7db4',
'address': {'slot': '0x03', 'bus': '0x00', 'domain': '0x0000', 'type':
'pci', 'function': '0x0'}, 'device': 'bridge', 'type': 'interface',
'name': 'vnet0'}, {'index': '2', 'iface': 'ide', 'name': 'hdc', 'al
ias': 'ide0-1-0', 'specParams': {}, 'readonly': 'True', 'deviceId':
'e738b50b-c200-4429-8489-4519325339c7', 'address': {'bus': '1',
'controller': '0', 'type': 'drive', 'target': '0', 'unit': '0'},
'device': 'cdrom', 'shared': 'false', 'path': '', 'type': 'disk'},
{'poolI
D': '00000000-0000-0000-0000-000000000000', 'volumeInfo': {'path':
'engine/2e2820f3-8c3d-487d-9a56-1b8cd278ec6c/images/5deeac2d-18d7-4622-9371-ebf965d2bd6b/60aa51b7-32eb-41a9-940d-9489b0375a3d',
'protocol': 'gluster', 'hosts': [{'port': '0', 'transport': 'tcp',
'name': '
virt1'}, {'port': '0', 'transport': 'tcp', 'name': 'virt2'}, {'port':
'0', 'transport': 'tcp', 'name': 'virt3'}]}, 'index': '0', 'iface':
'virtio', 'apparentsize': '62277025792', 'specParams': {}, 'imageID':
'5deeac2d-18d7-4622-9371-ebf965d2bd6b', 'readonly': 'False', 's
hared': 'exclusive', 'truesize': '3255476224', 'type': 'disk',
'domainID': '2e2820f3-8c3d-487d-9a56-1b8cd278ec6c', 'reqsize': '0',
'format': 'raw', 'deviceId': '5deeac2d-18d7-4622-9371-ebf965d2bd6b',
'address': {'slot': '0x06', 'bus': '0x00', 'domain': '0x0000', 'type':
'pci', 'function': '0x0'}, 'device': 'disk', 'path':
'/var/run/vdsm/storage/2e2820f3-8c3d-487d-9a56-1b8cd278ec6c/5deeac2d-18d7-4622-9371-ebf965d2bd6b/60aa51b7-32eb-41a9-940d-9489b0375a3d',
'propagateErrors': 'off', 'optional': 'false', 'name': 'vda',
'bootOrder': '1', 'v
olumeID': '60aa51b7-32eb-41a9-940d-9489b0375a3d', 'alias':
'virtio-disk0', 'volumeChain': [{'domainID':
'2e2820f3-8c3d-487d-9a56-1b8cd278ec6c', 'leaseOffset': 0, 'volumeID':
'60aa51b7-32eb-41a9-940d-9489b0375a3d', 'leasePath':
'/rhev/data-center/mnt/glusterSD/virt1:_engi
ne/2e2820f3-8c3d-487d-9a56-1b8cd278ec6c/images/5deeac2d-18d7-4622-9371-ebf965d2bd6b/60aa51b7-32eb-41a9-940d-9489b0375a3d.lease',
'imageID': '5deeac2d-18d7-4622-9371-ebf965d2bd6b', 'path':
'/rhev/data-center/mnt/glusterSD/virt1:_engine/2e2820f3-8c3d-487d-9a56-1b8cd278ec6c
/images/5deeac2d-18d7-4622-9371-ebf965d2bd6b/60aa51b7-32eb-41a9-940d-9489b0375a3d'}]}]
guestDiskMapping = {'5deeac2d-18d7-4622-9': {'name':
'/dev/vda'}, 'QEMU_DVD-ROM_QM00003': {'name': '/dev/sr0'}}
vmType = kvm
display = vnc
memSize = 16384
cpuType = Westmere
spiceSecureChannels =
smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
smp = 4
vmName = HostedEngine
clientIp =
maxVCpus = 16
[root@virt3 ~]#
[root@virt3 ~]# hosted-engine --vm-status
!! Cluster is in GLOBAL MAINTENANCE mode !!
--== Host 1 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : virt1.management.gnc.am
Host ID : 1
Engine status : {"reason": "vm not running on
this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : ef49e5b4
local_conf_timestamp : 7515
Host timestamp : 7512
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=7512 (Thu Aug 31 15:14:59 2017)
host-id=1
score=3400
vm_conf_refresh_time=7515 (Thu Aug 31 15:15:01 2017)
conf_on_shared_storage=True
maintenance=False
state=GlobalMaintenance
stopped=False
--== Host 3 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : virt3
Host ID : 3
Engine status : {"health": "good", "vm": "up",
"detail": "up"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 4a85111c
local_conf_timestamp : 102896
Host timestamp : 102893
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=102893 (Thu Aug 31 15:14:46 2017)
host-id=3
score=3400
vm_conf_refresh_time=102896 (Thu Aug 31 15:14:49 2017)
conf_on_shared_storage=True
maintenance=False
state=GlobalMaintenance
stopped=False
!! Cluster is in GLOBAL MAINTENANCE mode !!
Also my storage domain for hosted engine is inactive, can't activate
it it gives this error in web console:
VDSM command GetImagesListVDS failed: Storage domain does not exist:
(u'c44343af-cc4a-4bb7-a548-0c6f609d60d5',)
It seems I should fiddle with database a bit more, but is't scary thing for me.
Any help?
Best regards,
Misak Khachatryan
