[v2 1/1] Virtual machine migration
Issue: Migraton SSH authentication, even 2 peers have ssh key exchanged, migration still failed due to failed authentication, first of all, still need passwd input to get a remote libvirt connection second, after that, migration still failed with error: libvirt: QEMU Driver error : 操作失败: Failed to connect to remote libvirt URI qemu+ssh://root@9.115.122.75/system: Cannot recv data: Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-keyex,gssapi-with-mic,password).: Connection reset by peer How to test: Apply this patch(did some hack to easy debug) and issue migrate action with: sudo curl -k -u use:password -H "Content-Type: application/json" -H "Accept: application/json" https://localhost:8001/vms/RHEL-7/migrate -X POST -d '{"remoteHost": "9.115.122.75", "user": "root", "passwd": "passwd"}' v2: - use generate_action_handler_task - Fix args in generate_action_handler_task - Remove MigrationError - Put import in alphabetic - Support offline migration Signed-off-by: Simon Jin <simonjin@linux.vnet.ibm.com> --- docs/API.md | 4 ++ src/kimchi/control/vms.py | 1 + src/kimchi/i18n.py | 10 ++++ src/kimchi/model/vms.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+) diff --git a/docs/API.md b/docs/API.md index 9c06f85..36e5ab6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -132,6 +132,10 @@ the following general conventions: It emulates the power reset button on a machine. Note that there is a risk of data loss caused by reset without the guest OS shutdown. * connect: Prepare the connection for spice or vnc +* migrate: Migrate a virtual machine to a remote server, only support live mode without block migration. + * remoteHost: IP address or hostname of the remote server. + * user: user of the remote server. + * passwd: passwd of user on remote server. ### Sub-resource: Virtual Machine Screenshot diff --git a/src/kimchi/control/vms.py b/src/kimchi/control/vms.py index 88d8a81..0ab239e 100644 --- a/src/kimchi/control/vms.py +++ b/src/kimchi/control/vms.py @@ -42,6 +42,7 @@ class VM(Resource): for ident, node in sub_nodes.items(): setattr(self, ident, node(model, self.ident)) self.start = self.generate_action_handler('start') + self.migrate = self.generate_action_handler_task('migrate', ['remoteHost', 'user', 'passwd']) self.poweroff = self.generate_action_handler('poweroff') self.shutdown = self.generate_action_handler('shutdown') self.reset = self.generate_action_handler('reset') diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index f789259..6edc595 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -101,6 +101,16 @@ messages = { "KCHVM0030E": _("Unable to get access metadata of virtual machine %(name)s. Details: %(err)s"), "KCHVM0031E": _("The guest console password must be a string."), "KCHVM0032E": _("The life time for the guest console password must be a number."), + "KCHVM0033E": _("Can't migrate to localhost."), + "KCHVM0034E": _("Can't migrate vm %(name)s on Hypervisor arch %(srcarch)s.to a different Hypervisor arch %(destarch)s."), + "KCHVM0035E": _("Can't migrate from %(srchyp)s to a different Hypervisor type %(desthyp)s."), + "KCHVM0037E": _("Can't migrate vm %(name)s,which isn't in running state."), + "KCHVM0038E": _("Can't migrate vm %(name)s, vm has passthrough device %(hostdevs)s."), + "KCHVM0039E": _("Can't migrate vm %(name)s, vm already exist on destination server %(remoteHost)s."), + "KCHVM0040E": _("Failed Migrate vm %(name)s due error: %(err)s"), + "KCHVM0041E": _("Failed resume vm %(name)s from migration recover. Details: %(err)s"), + "KCHVM0042E": _("Failed destory vm %(name)s from migration recover. Details: %(err)s"), + "KCHVM0043E": _("Failed disconnect %(remoteuri)s when finish migrate vm %(name)s. Details: %(err)s"), "KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), diff --git a/src/kimchi/model/vms.py b/src/kimchi/model/vms.py index f2e4ae3..c6709ca 100644 --- a/src/kimchi/model/vms.py +++ b/src/kimchi/model/vms.py @@ -30,16 +30,20 @@ from xml.etree import ElementTree import libvirt from cherrypy.process.plugins import BackgroundTask +from kimchi.utils import add_task from kimchi import vnc from kimchi.config import READONLY_POOL_TYPE from kimchi.exception import InvalidOperation, InvalidParameter +from kimchi.model.libvirtconnection import LibvirtConnection from kimchi.exception import NotFoundError, OperationFailed from kimchi.model.config import CapabilitiesModel from kimchi.model.templates import TemplateModel +from kimchi.model.templates import TemplateModel from kimchi.model.utils import get_vm_name from kimchi.model.utils import get_metadata_node from kimchi.model.utils import set_metadata_node from kimchi.screenshot import VMScreenshot +#from kimchi.model.vmhostdevs import VMHostDevsModel from kimchi.utils import import_class, kimchi_log, run_setfacl_set_attr from kimchi.utils import template_name_from_uri from kimchi.xmlutils.utils import xpath_get_text, xml_item_update @@ -514,6 +518,147 @@ class VMModel(object): except libvirt.libvirtError as e: raise OperationFailed("KCHVM0019E", {'name': name, 'err': e.get_error_message()}) + + def _preCheck(self, name, destConn, remoteHost): + kimchi_log.debug('precheck migrate %s' % name) + + _destConn = destConn.get() + if remoteHost in ['localhost', '127.0.0.1']: + kimchi_log.debug('vm %s is not in running, only running vm can be migrated' % name) + raise OperationFailed("KCHVM00033E", {'name': name, + 'remoteHost': remoteHost}) + + + if self.conn.get().getInfo()[0] != _destConn.getInfo()[0]: + kimchi_log.debug('vm %s can not migrate to different arch server.' % (name, _destConn.getInfo()[0])) + raise OperationFailed("KCHVM00034E", {'name': name, + 'srcarch': self.conn.get().getType(), + 'destarch': _destConn.getType()}) + + if self.conn.get().getType() != _destConn.getType(): + kimchi_log.debug('vm %s can not migrate to a different hypervisor' % name) + raise OperationFailed("KCHVM00035E", {'name': self.name, + 'srchyp':self.conn.get().getType(), + 'desthyp': _destConn.getType()}) + + + #model.host import DOM_STATE_MAP, vmhostdevs import host, + #to void loop import DOM_STATE_MAP, move import VMHostDevsModel here + from kimchi.model.vmhostdevs import VMHostDevsModel + #check if there is any passthrough devices belong to this vm + vmhostdevs = VMHostDevsModel(conn = self.conn) + hostdevs = vmhostdevs.get_list(name) + if hostdevs: + raise OperationFailed("KCHVM00038E", {'name' : name, + 'hostdevs': hostdevs}) + #try: #FIXME + # self.get_vm(name, destConn) + #except NotFoundError as e: + # if e != NotFoundError: + # kimchi_log.debug('migrate vm %s already exist on %s' % (name, destConn.getURI())) + #raise OperationFailed("KCHVM00039E", {'name' : name, + # 'remoteHost': remoteHost}) + + def _getRemoteConn(self, remoteHost, user): + #open destination connect to migrate + transport = 'ssh' + duri = 'qemu+%s://%s@%s/system' % (transport, + user,remoteHost) + + conn = LibvirtConnection(duri) + return conn + + def _preMigrate(self, name): + #can't update VM while in Migration source/destination + kimchi_log.debug('prepare migrate %s' % name) + dom = self.get_vm(name, self.conn) + try: + dom.migrateSetMaxDowntime(0) + except libvirt.libvirtError as e: + print e.message + + def _do_migrate(self, name, destConn, remoteHost, cb): + _destConn = destConn.get() + cb('starting a migration') + kimchi_log.debug('migrate %s start' % name) + #import pdb + #pdb.set_trace() + dom = self.get_vm(name, self.conn) + flag = 0 + flag |= (libvirt.VIR_MIGRATE_PEER2PEER | + libvirt.VIR_MIGRATE_TUNNELLED) + + state = DOM_STATE_MAP[dom.info()[0]] + if state == 'running': + flag |= libvirt.VIR_MIGRATE_LIVE + try: + dom.migrateSetMaxDowntime(0) + except libvirt.libvirtError as e: + print e.message #FIXME + + elif state == 'shutoff': + flag |= libvirt.VIR_MIGRATE_OFFLINE + else: #FIXME any other state will prevend migration ? + kimchi_log.debug('Can not migrate vm %s when its in %s.' % (name, state)) + raise OperationFailed("KCHVM00037E", {'name': self.name}) + + try: + dom.migrate(_destConn, flag, None, None, 0) + except libvirt.libvirtError as e: + kimchi_log.error('migrate %s to %s failed' % (name, + remoteHost)) + self._recover(cb) + raise OperationFailed('KCHPOOL0040E', {'err': e.message, + 'name': name}) + finally: + self._finish(name, destConn, cb) + + def migrate(self, name, remoteHost, user, passwd ): + destconn = self._getRemoteConn(remoteHost, user) + + flag = self._preCheck(name, destconn, remoteHost) + self._preMigrate(name) + def cb(message): + print message + + + self._do_migrate(name, destconn, remoteHost,cb) + + task_id = add_task('/vms/%s/migrate' % name, self._do_migrate, self.objstore, + {'name' : name, 'destconn': destconn, + 'remoteHost': remoteHost}) + + # Record vm migrate task for future querying + try: + with self.objstore as session: + session.store('migrate', name, task_id) + return task_id + except Exception as e: + raise OperationFailed('KCHPOOL0037E', {'err': e.message}) + + def _recover(self, name, destconn, cb): + destvm = self.get_vm(name, destconn) + #recover + cb('recover from a failed migration',False) + kimchi_log.debug('recover from a failed migrate %s' % name) + try: + destvm.destroy() + kimchi_log.error("destory migrate vm on destination server") + except libvirt.libvirtError as e: + raise OperationFailed('KCHPOOL0042E', {'name' : name, + 'err': e.message}) + + def _finish(self, name, destConn, cb): + _destConn = destConn.get() + kimchi_log.debug('finished migrate %s' % name) + try: + _destConn.close() + except libvirt.libvirtError as e: + raise OperationFailed('KCHPOOL0043E', {'name' : name, + 'remoteuri' : _destConn.getURI(), + 'err': e.message}) + finally: + cb('Migrate finished', True) def poweroff(self, name): dom = self.get_vm(name, self.conn) -- 1.8.3.1
于 2014年11月11日 12:37, simonjin@linux.vnet.ibm.com 写道:
From: Simon Jin <simonjin@linux.vnet.ibm.com>
Issue: Migraton SSH authentication, even 2 peers have ssh key exchanged, migration still failed due to failed authentication, first of all, still need passwd input to get a remote libvirt connection second, after that, migration still failed with error: libvirt: QEMU Driver error : 操作失败: Failed to connect to remote libvirt URI qemu+ssh://root@9.115.122.75/system: Cannot recv data: Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-keyex,gssapi-with-mic,password).: Connection reset by peer The issue is caused by lunch kimchid with sudo, and ssh will then use ssh key under 'simon' but the real userid is root, like simon: sudo ssh root@9.115.122.75
virt-manager use libvirt.openAuth to pass password which from user input to libvir like below, but it doesn't work well as expected, the _auth_cb doesn't get called: import libvirt import os def open(_open_uri, passwordcb): def _auth_cb(creds, (passwordcb, passwordcreds)): for cred in creds: if cred[0] not in passwordcreds: raise RuntimeError("Unknown cred type '%s', expected only " "%s" % (cred[0], passwordcreds)) return passwordcb(creds) open_flags = 0 valid_auth_options = [libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE] authcb = _auth_cb authcb_data = passwordcb conn = libvirt.openAuth(_open_uri, [valid_auth_options, authcb, (authcb_data, valid_auth_options)], open_flags) print conn.getInfo() print conn.listDomainsID() return conn conn = open('qemu+ssh://simon@9.115.122.75/system','passwd')
How to test: Apply this patch(did some hack to easy debug) and issue migrate action with: sudo curl -k -u use:password -H "Content-Type: application/json" -H "Accept: application/json" https://localhost:8001/vms/RHEL-7/migrate -X POST -d '{"remoteHost": "9.115.122.75", "user": "root", "passwd": "passwd"}'
v2: - use generate_action_handler_task - Fix args in generate_action_handler_task - Remove MigrationError - Put import in alphabetic - Support offline migration
Signed-off-by: Simon Jin <simonjin@linux.vnet.ibm.com> --- docs/API.md | 4 ++ src/kimchi/control/vms.py | 1 + src/kimchi/i18n.py | 10 ++++ src/kimchi/model/vms.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+)
diff --git a/docs/API.md b/docs/API.md index 9c06f85..36e5ab6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -132,6 +132,10 @@ the following general conventions: It emulates the power reset button on a machine. Note that there is a risk of data loss caused by reset without the guest OS shutdown. * connect: Prepare the connection for spice or vnc +* migrate: Migrate a virtual machine to a remote server, only support live mode without block migration. + * remoteHost: IP address or hostname of the remote server. + * user: user of the remote server. + * passwd: passwd of user on remote server.
### Sub-resource: Virtual Machine Screenshot
diff --git a/src/kimchi/control/vms.py b/src/kimchi/control/vms.py index 88d8a81..0ab239e 100644 --- a/src/kimchi/control/vms.py +++ b/src/kimchi/control/vms.py @@ -42,6 +42,7 @@ class VM(Resource): for ident, node in sub_nodes.items(): setattr(self, ident, node(model, self.ident)) self.start = self.generate_action_handler('start') + self.migrate = self.generate_action_handler_task('migrate', ['remoteHost', 'user', 'passwd']) self.poweroff = self.generate_action_handler('poweroff') self.shutdown = self.generate_action_handler('shutdown') self.reset = self.generate_action_handler('reset') diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index f789259..6edc595 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -101,6 +101,16 @@ messages = { "KCHVM0030E": _("Unable to get access metadata of virtual machine %(name)s. Details: %(err)s"), "KCHVM0031E": _("The guest console password must be a string."), "KCHVM0032E": _("The life time for the guest console password must be a number."), + "KCHVM0033E": _("Can't migrate to localhost."), + "KCHVM0034E": _("Can't migrate vm %(name)s on Hypervisor arch %(srcarch)s.to a different Hypervisor arch %(destarch)s."), + "KCHVM0035E": _("Can't migrate from %(srchyp)s to a different Hypervisor type %(desthyp)s."), + "KCHVM0037E": _("Can't migrate vm %(name)s,which isn't in running state."), + "KCHVM0038E": _("Can't migrate vm %(name)s, vm has passthrough device %(hostdevs)s."), + "KCHVM0039E": _("Can't migrate vm %(name)s, vm already exist on destination server %(remoteHost)s."), + "KCHVM0040E": _("Failed Migrate vm %(name)s due error: %(err)s"), + "KCHVM0041E": _("Failed resume vm %(name)s from migration recover. Details: %(err)s"), + "KCHVM0042E": _("Failed destory vm %(name)s from migration recover. Details: %(err)s"), + "KCHVM0043E": _("Failed disconnect %(remoteuri)s when finish migrate vm %(name)s. Details: %(err)s"),
"KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), diff --git a/src/kimchi/model/vms.py b/src/kimchi/model/vms.py index f2e4ae3..c6709ca 100644 --- a/src/kimchi/model/vms.py +++ b/src/kimchi/model/vms.py @@ -30,16 +30,20 @@ from xml.etree import ElementTree import libvirt from cherrypy.process.plugins import BackgroundTask
+from kimchi.utils import add_task from kimchi import vnc from kimchi.config import READONLY_POOL_TYPE from kimchi.exception import InvalidOperation, InvalidParameter +from kimchi.model.libvirtconnection import LibvirtConnection from kimchi.exception import NotFoundError, OperationFailed from kimchi.model.config import CapabilitiesModel from kimchi.model.templates import TemplateModel +from kimchi.model.templates import TemplateModel from kimchi.model.utils import get_vm_name from kimchi.model.utils import get_metadata_node from kimchi.model.utils import set_metadata_node from kimchi.screenshot import VMScreenshot +#from kimchi.model.vmhostdevs import VMHostDevsModel from kimchi.utils import import_class, kimchi_log, run_setfacl_set_attr from kimchi.utils import template_name_from_uri from kimchi.xmlutils.utils import xpath_get_text, xml_item_update @@ -514,6 +518,147 @@ class VMModel(object): except libvirt.libvirtError as e: raise OperationFailed("KCHVM0019E", {'name': name, 'err': e.get_error_message()}) + + def _preCheck(self, name, destConn, remoteHost): + kimchi_log.debug('precheck migrate %s' % name) + + _destConn = destConn.get() + if remoteHost in ['localhost', '127.0.0.1']: + kimchi_log.debug('vm %s is not in running, only running vm can be migrated' % name) + raise OperationFailed("KCHVM00033E", {'name': name, + 'remoteHost': remoteHost}) + + + if self.conn.get().getInfo()[0] != _destConn.getInfo()[0]: + kimchi_log.debug('vm %s can not migrate to different arch server.' % (name, _destConn.getInfo()[0])) + raise OperationFailed("KCHVM00034E", {'name': name, + 'srcarch': self.conn.get().getType(), + 'destarch': _destConn.getType()}) + + if self.conn.get().getType() != _destConn.getType(): + kimchi_log.debug('vm %s can not migrate to a different hypervisor' % name) + raise OperationFailed("KCHVM00035E", {'name': self.name, + 'srchyp':self.conn.get().getType(), + 'desthyp': _destConn.getType()}) + + + #model.host import DOM_STATE_MAP, vmhostdevs import host, + #to void loop import DOM_STATE_MAP, move import VMHostDevsModel here + from kimchi.model.vmhostdevs import VMHostDevsModel + #check if there is any passthrough devices belong to this vm + vmhostdevs = VMHostDevsModel(conn = self.conn) + hostdevs = vmhostdevs.get_list(name) + if hostdevs: + raise OperationFailed("KCHVM00038E", {'name' : name, + 'hostdevs': hostdevs}) + #try: #FIXME + # self.get_vm(name, destConn) + #except NotFoundError as e: + # if e != NotFoundError: + # kimchi_log.debug('migrate vm %s already exist on %s' % (name, destConn.getURI())) + #raise OperationFailed("KCHVM00039E", {'name' : name, + # 'remoteHost': remoteHost}) + + def _getRemoteConn(self, remoteHost, user): + #open destination connect to migrate + transport = 'ssh' + duri = 'qemu+%s://%s@%s/system' % (transport, + user,remoteHost) + + conn = LibvirtConnection(duri) + return conn + + def _preMigrate(self, name): + #can't update VM while in Migration source/destination + kimchi_log.debug('prepare migrate %s' % name) + dom = self.get_vm(name, self.conn) + try: + dom.migrateSetMaxDowntime(0) + except libvirt.libvirtError as e: + print e.message + + def _do_migrate(self, name, destConn, remoteHost, cb): + _destConn = destConn.get() + cb('starting a migration') + kimchi_log.debug('migrate %s start' % name) + #import pdb + #pdb.set_trace() + dom = self.get_vm(name, self.conn) + flag = 0 + flag |= (libvirt.VIR_MIGRATE_PEER2PEER | + libvirt.VIR_MIGRATE_TUNNELLED) + + state = DOM_STATE_MAP[dom.info()[0]] + if state == 'running': + flag |= libvirt.VIR_MIGRATE_LIVE + try: + dom.migrateSetMaxDowntime(0) + except libvirt.libvirtError as e: + print e.message #FIXME + + elif state == 'shutoff': + flag |= libvirt.VIR_MIGRATE_OFFLINE + else: #FIXME any other state will prevend migration ? + kimchi_log.debug('Can not migrate vm %s when its in %s.' % (name, state)) + raise OperationFailed("KCHVM00037E", {'name': self.name}) + + try: + dom.migrate(_destConn, flag, None, None, 0) + except libvirt.libvirtError as e: + kimchi_log.error('migrate %s to %s failed' % (name, + remoteHost)) + self._recover(cb) + raise OperationFailed('KCHPOOL0040E', {'err': e.message, + 'name': name}) + finally: + self._finish(name, destConn, cb) + + def migrate(self, name, remoteHost, user, passwd ): + destconn = self._getRemoteConn(remoteHost, user) + + flag = self._preCheck(name, destconn, remoteHost) + self._preMigrate(name) + def cb(message): + print message + + + self._do_migrate(name, destconn, remoteHost,cb) + + task_id = add_task('/vms/%s/migrate' % name, self._do_migrate, self.objstore, + {'name' : name, 'destconn': destconn, + 'remoteHost': remoteHost}) + + # Record vm migrate task for future querying + try: + with self.objstore as session: + session.store('migrate', name, task_id) + return task_id + except Exception as e: + raise OperationFailed('KCHPOOL0037E', {'err': e.message}) + + def _recover(self, name, destconn, cb): + destvm = self.get_vm(name, destconn) + #recover + cb('recover from a failed migration',False) + kimchi_log.debug('recover from a failed migrate %s' % name) + try: + destvm.destroy() + kimchi_log.error("destory migrate vm on destination server") + except libvirt.libvirtError as e: + raise OperationFailed('KCHPOOL0042E', {'name' : name, + 'err': e.message}) + + def _finish(self, name, destConn, cb): + _destConn = destConn.get() + kimchi_log.debug('finished migrate %s' % name) + try: + _destConn.close() + except libvirt.libvirtError as e: + raise OperationFailed('KCHPOOL0043E', {'name' : name, + 'remoteuri' : _destConn.getURI(), + 'err': e.message}) + finally: + cb('Migrate finished', True)
def poweroff(self, name): dom = self.get_vm(name, self.conn)
Looks like ssh-agent can do the trick, however i didn't make it work. http://www.sudo.ws/pipermail/sudo-users/2003-April/001479.html 于 2014年11月11日 13:27, simonjin 写道:
于 2014年11月11日 12:37, simonjin@linux.vnet.ibm.com 写道:
From: Simon Jin <simonjin@linux.vnet.ibm.com>
Issue: Migraton SSH authentication, even 2 peers have ssh key exchanged, migration still failed due to failed authentication, first of all, still need passwd input to get a remote libvirt connection second, after that, migration still failed with error: libvirt: QEMU Driver error : 操作失败: Failed to connect to remote libvirt URI qemu+ssh://root@9.115.122.75/system: Cannot recv data: Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-keyex,gssapi-with-mic,password).: Connection reset by peer The issue is caused by lunch kimchid with sudo, and ssh will then use ssh key under 'simon' but the real userid is root, like simon: sudo ssh root@9.115.122.75
virt-manager use libvirt.openAuth to pass password which from user input to libvir like below, but it doesn't work well as expected, the _auth_cb doesn't get called: import libvirt import os
def open(_open_uri, passwordcb): def _auth_cb(creds, (passwordcb, passwordcreds)): for cred in creds: if cred[0] not in passwordcreds: raise RuntimeError("Unknown cred type '%s', expected only " "%s" % (cred[0], passwordcreds)) return passwordcb(creds)
open_flags = 0 valid_auth_options = [libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE]
authcb = _auth_cb authcb_data = passwordcb
conn = libvirt.openAuth(_open_uri, [valid_auth_options, authcb, (authcb_data, valid_auth_options)], open_flags)
print conn.getInfo() print conn.listDomainsID() return conn
conn = open('qemu+ssh://simon@9.115.122.75/system','passwd')
How to test: Apply this patch(did some hack to easy debug) and issue migrate action with: sudo curl -k -u use:password -H "Content-Type: application/json" -H "Accept: application/json" https://localhost:8001/vms/RHEL-7/migrate -X POST -d '{"remoteHost": "9.115.122.75", "user": "root", "passwd": "passwd"}'
v2: - use generate_action_handler_task - Fix args in generate_action_handler_task - Remove MigrationError - Put import in alphabetic - Support offline migration
Signed-off-by: Simon Jin <simonjin@linux.vnet.ibm.com> --- docs/API.md | 4 ++ src/kimchi/control/vms.py | 1 + src/kimchi/i18n.py | 10 ++++ src/kimchi/model/vms.py | 145 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+)
diff --git a/docs/API.md b/docs/API.md index 9c06f85..36e5ab6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -132,6 +132,10 @@ the following general conventions: It emulates the power reset button on a machine. Note that there is a risk of data loss caused by reset without the guest OS shutdown. * connect: Prepare the connection for spice or vnc +* migrate: Migrate a virtual machine to a remote server, only support live mode without block migration. + * remoteHost: IP address or hostname of the remote server. + * user: user of the remote server. + * passwd: passwd of user on remote server. ### Sub-resource: Virtual Machine Screenshot diff --git a/src/kimchi/control/vms.py b/src/kimchi/control/vms.py index 88d8a81..0ab239e 100644 --- a/src/kimchi/control/vms.py +++ b/src/kimchi/control/vms.py @@ -42,6 +42,7 @@ class VM(Resource): for ident, node in sub_nodes.items(): setattr(self, ident, node(model, self.ident)) self.start = self.generate_action_handler('start') + self.migrate = self.generate_action_handler_task('migrate', ['remoteHost', 'user', 'passwd']) self.poweroff = self.generate_action_handler('poweroff') self.shutdown = self.generate_action_handler('shutdown') self.reset = self.generate_action_handler('reset') diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index f789259..6edc595 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -101,6 +101,16 @@ messages = { "KCHVM0030E": _("Unable to get access metadata of virtual machine %(name)s. Details: %(err)s"), "KCHVM0031E": _("The guest console password must be a string."), "KCHVM0032E": _("The life time for the guest console password must be a number."), + "KCHVM0033E": _("Can't migrate to localhost."), + "KCHVM0034E": _("Can't migrate vm %(name)s on Hypervisor arch %(srcarch)s.to a different Hypervisor arch %(destarch)s."), + "KCHVM0035E": _("Can't migrate from %(srchyp)s to a different Hypervisor type %(desthyp)s."), + "KCHVM0037E": _("Can't migrate vm %(name)s,which isn't in running state."), + "KCHVM0038E": _("Can't migrate vm %(name)s, vm has passthrough device %(hostdevs)s."), + "KCHVM0039E": _("Can't migrate vm %(name)s, vm already exist on destination server %(remoteHost)s."), + "KCHVM0040E": _("Failed Migrate vm %(name)s due error: %(err)s"), + "KCHVM0041E": _("Failed resume vm %(name)s from migration recover. Details: %(err)s"), + "KCHVM0042E": _("Failed destory vm %(name)s from migration recover. Details: %(err)s"), + "KCHVM0043E": _("Failed disconnect %(remoteuri)s when finish migrate vm %(name)s. Details: %(err)s"), "KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), diff --git a/src/kimchi/model/vms.py b/src/kimchi/model/vms.py index f2e4ae3..c6709ca 100644 --- a/src/kimchi/model/vms.py +++ b/src/kimchi/model/vms.py @@ -30,16 +30,20 @@ from xml.etree import ElementTree import libvirt from cherrypy.process.plugins import BackgroundTask +from kimchi.utils import add_task from kimchi import vnc from kimchi.config import READONLY_POOL_TYPE from kimchi.exception import InvalidOperation, InvalidParameter +from kimchi.model.libvirtconnection import LibvirtConnection from kimchi.exception import NotFoundError, OperationFailed from kimchi.model.config import CapabilitiesModel from kimchi.model.templates import TemplateModel +from kimchi.model.templates import TemplateModel from kimchi.model.utils import get_vm_name from kimchi.model.utils import get_metadata_node from kimchi.model.utils import set_metadata_node from kimchi.screenshot import VMScreenshot +#from kimchi.model.vmhostdevs import VMHostDevsModel from kimchi.utils import import_class, kimchi_log, run_setfacl_set_attr from kimchi.utils import template_name_from_uri from kimchi.xmlutils.utils import xpath_get_text, xml_item_update @@ -514,6 +518,147 @@ class VMModel(object): except libvirt.libvirtError as e: raise OperationFailed("KCHVM0019E", {'name': name, 'err': e.get_error_message()}) + + def _preCheck(self, name, destConn, remoteHost): + kimchi_log.debug('precheck migrate %s' % name) + + _destConn = destConn.get() + if remoteHost in ['localhost', '127.0.0.1']: + kimchi_log.debug('vm %s is not in running, only running vm can be migrated' % name) + raise OperationFailed("KCHVM00033E", {'name': name, + 'remoteHost': remoteHost}) + + + if self.conn.get().getInfo()[0] != _destConn.getInfo()[0]: + kimchi_log.debug('vm %s can not migrate to different arch server.' % (name, _destConn.getInfo()[0])) + raise OperationFailed("KCHVM00034E", {'name': name, + 'srcarch': self.conn.get().getType(), + 'destarch': _destConn.getType()}) + + if self.conn.get().getType() != _destConn.getType(): + kimchi_log.debug('vm %s can not migrate to a different hypervisor' % name) + raise OperationFailed("KCHVM00035E", {'name': self.name, + 'srchyp':self.conn.get().getType(), + 'desthyp': _destConn.getType()}) + + + #model.host import DOM_STATE_MAP, vmhostdevs import host, + #to void loop import DOM_STATE_MAP, move import VMHostDevsModel here + from kimchi.model.vmhostdevs import VMHostDevsModel + #check if there is any passthrough devices belong to this vm + vmhostdevs = VMHostDevsModel(conn = self.conn) + hostdevs = vmhostdevs.get_list(name) + if hostdevs: + raise OperationFailed("KCHVM00038E", {'name' : name, + 'hostdevs': hostdevs}) + #try: #FIXME + # self.get_vm(name, destConn) + #except NotFoundError as e: + # if e != NotFoundError: + # kimchi_log.debug('migrate vm %s already exist on %s' % (name, destConn.getURI())) + #raise OperationFailed("KCHVM00039E", {'name' : name, + # 'remoteHost': remoteHost}) + + def _getRemoteConn(self, remoteHost, user): + #open destination connect to migrate + transport = 'ssh' + duri = 'qemu+%s://%s@%s/system' % (transport, + user,remoteHost) + + conn = LibvirtConnection(duri) + return conn + + def _preMigrate(self, name): + #can't update VM while in Migration source/destination + kimchi_log.debug('prepare migrate %s' % name) + dom = self.get_vm(name, self.conn) + try: + dom.migrateSetMaxDowntime(0) + except libvirt.libvirtError as e: + print e.message + + def _do_migrate(self, name, destConn, remoteHost, cb): + _destConn = destConn.get() + cb('starting a migration') + kimchi_log.debug('migrate %s start' % name) + #import pdb + #pdb.set_trace() + dom = self.get_vm(name, self.conn) + flag = 0 + flag |= (libvirt.VIR_MIGRATE_PEER2PEER | + libvirt.VIR_MIGRATE_TUNNELLED) + + state = DOM_STATE_MAP[dom.info()[0]] + if state == 'running': + flag |= libvirt.VIR_MIGRATE_LIVE + try: + dom.migrateSetMaxDowntime(0) + except libvirt.libvirtError as e: + print e.message #FIXME + + elif state == 'shutoff': + flag |= libvirt.VIR_MIGRATE_OFFLINE + else: #FIXME any other state will prevend migration ? + kimchi_log.debug('Can not migrate vm %s when its in %s.' % (name, state)) + raise OperationFailed("KCHVM00037E", {'name': self.name}) + + try: + dom.migrate(_destConn, flag, None, None, 0) + except libvirt.libvirtError as e: + kimchi_log.error('migrate %s to %s failed' % (name, + remoteHost)) + self._recover(cb) + raise OperationFailed('KCHPOOL0040E', {'err': e.message, + 'name': name}) + finally: + self._finish(name, destConn, cb) + + def migrate(self, name, remoteHost, user, passwd ): + destconn = self._getRemoteConn(remoteHost, user) + + flag = self._preCheck(name, destconn, remoteHost) + self._preMigrate(name) + def cb(message): + print message + + + self._do_migrate(name, destconn, remoteHost,cb) + + task_id = add_task('/vms/%s/migrate' % name, self._do_migrate, self.objstore, + {'name' : name, 'destconn': destconn, + 'remoteHost': remoteHost}) + + # Record vm migrate task for future querying + try: + with self.objstore as session: + session.store('migrate', name, task_id) + return task_id + except Exception as e: + raise OperationFailed('KCHPOOL0037E', {'err': e.message}) + + def _recover(self, name, destconn, cb): + destvm = self.get_vm(name, destconn) + #recover + cb('recover from a failed migration',False) + kimchi_log.debug('recover from a failed migrate %s' % name) + try: + destvm.destroy() + kimchi_log.error("destory migrate vm on destination server") + except libvirt.libvirtError as e: + raise OperationFailed('KCHPOOL0042E', {'name' : name, + 'err': e.message}) + + def _finish(self, name, destConn, cb): + _destConn = destConn.get() + kimchi_log.debug('finished migrate %s' % name) + try: + _destConn.close() + except libvirt.libvirtError as e: + raise OperationFailed('KCHPOOL0043E', {'name' : name, + 'remoteuri' : _destConn.getURI(), + 'err': e.message}) + finally: + cb('Migrate finished', True) def poweroff(self, name): dom = self.get_vm(name, self.conn)
_______________________________________________ Kimchi-devel mailing list Kimchi-devel@ovirt.org http://lists.ovirt.org/mailman/listinfo/kimchi-devel
On 11-11-2014 02:37, simonjin@linux.vnet.ibm.com wrote:
+from kimchi.utils import add_task
There's already an import "kimchi.utils" below. Add the "add_task" first to that list.
from kimchi.exception import InvalidOperation, InvalidParameter +from kimchi.model.libvirtconnection import LibvirtConnection from kimchi.exception import NotFoundError, OperationFailed
Add that line in alphabetical order.
from kimchi.model.templates import TemplateModel +from kimchi.model.templates import TemplateModel
Isn't that class already imported on the line above?
from kimchi.model.utils import get_vm_name from kimchi.model.utils import get_metadata_node from kimchi.model.utils import set_metadata_node from kimchi.screenshot import VMScreenshot +#from kimchi.model.vmhostdevs import VMHostDevsModel
Unless that line is here due to debug status of this patch, there's no need to keep it commented.
+ if remoteHost in ['localhost', '127.0.0.1']: + kimchi_log.debug('vm %s is not in running, only running vm can be migrated' % name) + raise OperationFailed("KCHVM00033E", {'name': name, + 'remoteHost': remoteHost})
I don't think that is the reason why the VM cannot be migrated... By looking at the code and at the message KCHVM00033E, the reason is that the user is trying to migrate to localhost.
+ if self.conn.get().getInfo()[0] != _destConn.getInfo()[0]: + kimchi_log.debug('vm %s can not migrate to different arch server.' % (name, _destConn.getInfo()[0])) + raise OperationFailed("KCHVM00034E", {'name': name, + 'srcarch': self.conn.get().getType(), + 'destarch': _destConn.getType()})
'srcarch' and 'destarch' don't seem to contain the servers' architectures, but their hypervisor type. Also, only run "getInfo" (and other connection functions) once and store its value on a local variable, especially because we're dealing with remote connections and it might take a little more time to execute them.
+ + if self.conn.get().getType() != _destConn.getType(): + kimchi_log.debug('vm %s can not migrate to a different hypervisor' % name) + raise OperationFailed("KCHVM00035E", {'name': self.name, + 'srchyp':self.conn.get().getType(), + 'desthyp': _destConn.getType()})
As I commented earlier, only run "getType" once and store the values on local variables.
+ def _getRemoteConn(self, remoteHost, user): + #open destination connect to migrate + transport = 'ssh' + duri = 'qemu+%s://%s@%s/system' % (transport, + user,remoteHost) + + conn = LibvirtConnection(duri) + return conn
That function can be static, it doesn't use "self".
+ def _do_migrate(self, name, destConn, remoteHost, cb): + _destConn = destConn.get() + cb('starting a migration') + kimchi_log.debug('migrate %s start' % name) + #import pdb + #pdb.set_trace() + dom = self.get_vm(name, self.conn) + flag = 0 + flag |= (libvirt.VIR_MIGRATE_PEER2PEER | + libvirt.VIR_MIGRATE_TUNNELLED) + + state = DOM_STATE_MAP[dom.info()[0]] + if state == 'running': + flag |= libvirt.VIR_MIGRATE_LIVE + try: + dom.migrateSetMaxDowntime(0)
Isn't "migrateSetMaxDowntime" called in "_preMigrate", which is called before "_do_migrate"?
+ # Record vm migrate task for future querying + try: + with self.objstore as session: + session.store('migrate', name, task_id) + return task_id
If I'm not mistaken, the AsyncTask constructor (which is called inside the function "add_task" above) already registers the task in the local object store.
Thanks for the review. 于 2014年11月13日 03:22, Crístian Viana 写道:
On 11-11-2014 02:37, simonjin@linux.vnet.ibm.com wrote:
+from kimchi.utils import add_task
There's already an import "kimchi.utils" below. Add the "add_task" first to that list. Will fix.
from kimchi.exception import InvalidOperation, InvalidParameter +from kimchi.model.libvirtconnection import LibvirtConnection from kimchi.exception import NotFoundError, OperationFailed
Add that line in alphabetical order. Will fix.
from kimchi.model.templates import TemplateModel +from kimchi.model.templates import TemplateModel
Isn't that class already imported on the line above? Will fix.
from kimchi.model.utils import get_vm_name from kimchi.model.utils import get_metadata_node from kimchi.model.utils import set_metadata_node from kimchi.screenshot import VMScreenshot +#from kimchi.model.vmhostdevs import VMHostDevsModel
Unless that line is here due to debug status of this patch, there's no need to keep it commented.
Will fix.
+ if remoteHost in ['localhost', '127.0.0.1']: + kimchi_log.debug('vm %s is not in running, only running vm can be migrated' % name) + raise OperationFailed("KCHVM00033E", {'name': name, + 'remoteHost': remoteHost})
I don't think that is the reason why the VM cannot be migrated... By looking at the code and at the message KCHVM00033E, the reason is that the user is trying to migrate to localhost.
Will fix.
+ if self.conn.get().getInfo()[0] != _destConn.getInfo()[0]: + kimchi_log.debug('vm %s can not migrate to different arch server.' % (name, _destConn.getInfo()[0])) + raise OperationFailed("KCHVM00034E", {'name': name, + 'srcarch': self.conn.get().getType(), + 'destarch': _destConn.getType()})
'srcarch' and 'destarch' don't seem to contain the servers' architectures, but their hypervisor type.
_destConn.getInfo()[0] contains the servers' architectures like x86_64 or ppc.
Also, only run "getInfo" (and other connection functions) once and store its value on a local variable, especially because we're dealing with remote connections and it might take a little more time to execute them. OK.
+ + if self.conn.get().getType() != _destConn.getType(): + kimchi_log.debug('vm %s can not migrate to a different hypervisor' % name) + raise OperationFailed("KCHVM00035E", {'name': self.name, + 'srchyp':self.conn.get().getType(), + 'desthyp': _destConn.getType()})
As I commented earlier, only run "getType" once and store the values on local variables.
+ def _getRemoteConn(self, remoteHost, user): + #open destination connect to migrate + transport = 'ssh' + duri = 'qemu+%s://%s@%s/system' % (transport, + user,remoteHost) + + conn = LibvirtConnection(duri) + return conn
That function can be static, it doesn't use "self". OK.
+ def _do_migrate(self, name, destConn, remoteHost, cb): + _destConn = destConn.get() + cb('starting a migration') + kimchi_log.debug('migrate %s start' % name) + #import pdb + #pdb.set_trace() + dom = self.get_vm(name, self.conn) + flag = 0 + flag |= (libvirt.VIR_MIGRATE_PEER2PEER | + libvirt.VIR_MIGRATE_TUNNELLED) + + state = DOM_STATE_MAP[dom.info()[0]] + if state == 'running': + flag |= libvirt.VIR_MIGRATE_LIVE + try: + dom.migrateSetMaxDowntime(0)
Isn't "migrateSetMaxDowntime" called in "_preMigrate", which is called before "_do_migrate"?
Will clean this up.
+ # Record vm migrate task for future querying + try: + with self.objstore as session: + session.store('migrate', name, task_id) + return task_id
If I'm not mistaken, the AsyncTask constructor (which is called inside the function "add_task" above) already registers the task in the local object store. No, it doesn't .
-Simon
On 13-11-2014 04:46, simonjin wrote:
+ if self.conn.get().getInfo()[0] != _destConn.getInfo()[0]: + kimchi_log.debug('vm %s can not migrate to different arch server.' % (name, _destConn.getInfo()[0])) + raise OperationFailed("KCHVM00034E", {'name': name, + 'srcarch': self.conn.get().getType(), + 'destarch': _destConn.getType()})
'srcarch' and 'destarch' don't seem to contain the servers' architectures, but their hypervisor type.
_destConn.getInfo()[0] contains the servers' architectures like x86_64 or ppc.
Exactly, but that's not what the exception parameters 'srcarch' and 'destarch' have. They contain the return of "getType".
+ # Record vm migrate task for future querying + try: + with self.objstore as session: + session.store('migrate', name, task_id) + return task_id
If I'm not mistaken, the AsyncTask constructor (which is called inside the function "add_task" above) already registers the task in the local object store. No, it doesn't .
Please, take a look at the function "_save_helper" in src/kimchi/asynctask.py. It is called from the AsyncTask constructor and it registers the current task in the object store.
-Simon
participants (3)
-
Crístian Viana -
Simon Jin -
simonjin