Hello.
I'm on 4.1.3 with self hosted engine and glusterfs as storage.
I updated the kernel on engine so I executed these steps:
- enable global maintenace from the web admin gui
- wait some minutes
- shutdown the engine vm from inside its OS
- wait some minutes
- execute on one host
[root@ovirt02 ~]# hosted-engine --set-maintenance --mode=none
I see that the qemu-kvm process for the engine starts on two hosts and then on one of them it gets a "kill -15" and stops
Is it expected behaviour? It seems somehow dangerous to me..
- when in maintenance
[root@ovirt02 ~]# hosted-engine --vm-status
!! Cluster is in GLOBAL MAINTENANCE mode !!
--== Host 1 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt01.localdomain.local
Host ID : 1
Engine status : {"health": "good", "vm": "up", "detail": "up"}
Score : 2597
stopped : False
Local maintenance : False
crc32 : 7931c5c3
local_conf_timestamp : 19811
Host timestamp : 19794
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=19794 (Sun Jul 9 21:31:50 2017)
host-id=1
score=2597
vm_conf_refresh_time=19811 (Sun Jul 9 21:32:06 2017)
conf_on_shared_storage=True
maintenance=False
state=GlobalMaintenance
stopped=False
--== Host 2 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : 192.168.150.103
Host ID : 2
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 616ceb02
local_conf_timestamp : 2829
Host timestamp : 2812
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=2812 (Sun Jul 9 21:31:52 2017)
host-id=2
score=3400
vm_conf_refresh_time=2829 (Sun Jul 9 21:32:09 2017)
conf_on_shared_storage=True
maintenance=False
state=GlobalMaintenance
stopped=False
--== Host 3 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt03.localdomain.local
Host ID : 3
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 871204b2
local_conf_timestamp : 24584
Host timestamp : 24567
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=24567 (Sun Jul 9 21:31:52 2017)
host-id=3
score=3400
vm_conf_refresh_time=24584 (Sun Jul 9 21:32:09 2017)
conf_on_shared_storage=True
maintenance=False
state=GlobalMaintenance
stopped=False
!! Cluster is in GLOBAL MAINTENANCE mode !!
[root@ovirt02 ~]#
- then I exit global maintenance
[root@ovirt02 ~]# hosted-engine --set-maintenance --mode=none
- During monitoring of status, at some point I see "EngineStart" on both host2 and host3
[root@ovirt02 ~]# hosted-engine --vm-status
--== Host 1 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt01.localdomain.local
Host ID : 1
Engine status : {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "down"}
Score : 3230
stopped : False
Local maintenance : False
crc32 : 25cadbfb
local_conf_timestamp : 20055
Host timestamp : 20040
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=20040 (Sun Jul 9 21:35:55 2017)
host-id=1
score=3230
vm_conf_refresh_time=20055 (Sun Jul 9 21:36:11 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineDown
stopped=False
--== Host 2 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : 192.168.150.103
Host ID : 2
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : e6951128
local_conf_timestamp : 3075
Host timestamp : 3058
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=3058 (Sun Jul 9 21:35:59 2017)
host-id=2
score=3400
vm_conf_refresh_time=3075 (Sun Jul 9 21:36:15 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStart
stopped=False
--== Host 3 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt03.localdomain.local
Host ID : 3
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 382efde5
local_conf_timestamp : 24832
Host timestamp : 24816
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=24816 (Sun Jul 9 21:36:01 2017)
host-id=3
score=3400
vm_conf_refresh_time=24832 (Sun Jul 9 21:36:17 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStart
stopped=False
[root@ovirt02 ~]#
and then
[root@ovirt02 ~]# hosted-engine --vm-status
--== Host 1 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt01.localdomain.local
Host ID : 1
Engine status : {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "down"}
Score : 3253
stopped : False
Local maintenance : False
crc32 : 3fc39f31
local_conf_timestamp : 20087
Host timestamp : 20070
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=20070 (Sun Jul 9 21:36:26 2017)
host-id=1
score=3253
vm_conf_refresh_time=20087 (Sun Jul 9 21:36:43 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineDown
stopped=False
--== Host 2 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : 192.168.150.103
Host ID : 2
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 4a05c31e
local_conf_timestamp : 3109
Host timestamp : 3079
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=3079 (Sun Jul 9 21:36:19 2017)
host-id=2
score=3400
vm_conf_refresh_time=3109 (Sun Jul 9 21:36:49 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStarting
stopped=False
--== Host 3 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt03.localdomain.local
Host ID : 3
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 382efde5
local_conf_timestamp : 24832
Host timestamp : 24816
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=24816 (Sun Jul 9 21:36:01 2017)
host-id=3
score=3400
vm_conf_refresh_time=24832 (Sun Jul 9 21:36:17 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStart
stopped=False
[root@ovirt02 ~]#
and
[root@ovirt02 ~]# hosted-engine --vm-status
--== Host 1 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt01.localdomain.local
Host ID : 1
Engine status : {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "down"}
Score : 3253
stopped : False
Local maintenance : False
crc32 : 3fc39f31
local_conf_timestamp : 20087
Host timestamp : 20070
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=20070 (Sun Jul 9 21:36:26 2017)
host-id=1
score=3253
vm_conf_refresh_time=20087 (Sun Jul 9 21:36:43 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineDown
stopped=False
--== Host 2 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : 192.168.150.103
Host ID : 2
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 4a05c31e
local_conf_timestamp : 3109
Host timestamp : 3079
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=3079 (Sun Jul 9 21:36:19 2017)
host-id=2
score=3400
vm_conf_refresh_time=3109 (Sun Jul 9 21:36:49 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStarting
stopped=False
--== Host 3 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt03.localdomain.local
Host ID : 3
Engine status : {"reason": "vm not running on this host", "health": "bad", "vm": "down", "detail": "unknown"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : fc1e8cf9
local_conf_timestamp : 24868
Host timestamp : 24836
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=24836 (Sun Jul 9 21:36:21 2017)
host-id=3
score=3400
vm_conf_refresh_time=24868 (Sun Jul 9 21:36:53 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStarting
stopped=False
[root@ovirt02 ~]#
and at the end Host3 goes to "ForceStop" for the engine
[root@ovirt02 ~]# hosted-engine --vm-status
--== Host 1 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt01.localdomain.local
Host ID : 1
Engine status : {"reason": "bad vm status", "health": "bad", "vm": "down", "detail": "down"}
Score : 3312
stopped : False
Local maintenance : False
crc32 : e9d53432
local_conf_timestamp : 20120
Host timestamp : 20102
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=20102 (Sun Jul 9 21:36:58 2017)
host-id=1
score=3312
vm_conf_refresh_time=20120 (Sun Jul 9 21:37:15 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineDown
stopped=False
--== Host 2 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : 192.168.150.103
Host ID : 2
Engine status : {"reason": "bad vm status", "health": "bad", "vm": "up", "detail": "powering up"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 7d2330be
local_conf_timestamp : 3141
Host timestamp : 3124
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=3124 (Sun Jul 9 21:37:04 2017)
host-id=2
score=3400
vm_conf_refresh_time=3141 (Sun Jul 9 21:37:21 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineStarting
stopped=False
--== Host 3 status ==--
conf_on_shared_storage : True
Status up-to-date : True
Hostname : ovirt03.localdomain.local
Host ID : 3
Engine status : {"reason": "Storage of VM is locked. Is another host already starting the VM?", "health": "bad", "vm": "already_locked", "detail": "down"}
Score : 3400
stopped : False
Local maintenance : False
crc32 : 179825e8
local_conf_timestamp : 24900
Host timestamp : 24883
Extra metadata (valid at timestamp):
metadata_parse_version=1
metadata_feature_version=1
timestamp=24883 (Sun Jul 9 21:37:08 2017)
host-id=3
score=3400
vm_conf_refresh_time=24900 (Sun Jul 9 21:37:24 2017)
conf_on_shared_storage=True
maintenance=False
state=EngineForceStop
stopped=False
[root@ovirt02 ~]#
Comparing /var/log/libvirt/qemu/HostedEngine of host2 and host3
Host2:
2017-07-09 19:36:36.094+0000: starting up libvirt version: 2.0.0, package: 10.el7_3.9 (CentOS BuildSystem <
http://bugs.centos.org>, 2017-05-25-20:52:28,
c1bm.rdu2.centos.org), qemu version: 2.6.0 (qemu-kvm-ev-2.6.0-28.el7.10.1), hostname: ovirt02.localdomain.local
... char device redirected to /dev/pts/1 (label charconsole0)
warning: host doesn't support requested feature: CPUID.07H:EBX.erms [bit 9]
Host3:
2017-07-09 19:36:38.143+0000: starting up libvirt version: 2.0.0, package: 10.el7_3.9 (CentOS BuildSystem <
http://bugs.centos.org>, 2017-05-25-20:52:28,
c1bm.rdu2.centos.org), qemu version: 2.6.0 (qemu-kvm-ev-2.6.0-28.el7.10.1), hos
tname: ovirt03.localdomain.local
... char device redirected to /dev/pts/1 (label charconsole0)
2017-07-09 19:36:38.584+0000: shutting down
2017-07-09T19:36:38.589729Z qemu-kvm: terminating on signal 15 from pid 1835
any comment?
Is it only a matter of powering on the VM in paused mode before starting the OS itself, or do I risk corruption due to 2 qemu-kvm processes trying to start the engine vm os?
Thanks,
Gianluca