Re: [Kimchi-devel] [v6 1/1] Virtual machine migration

On 2014年12月16日 10:39, simonjin(a)linux.vnet.ibm.com wrote: > From: Simon Jin<simonjin(a)linux.vnet.ibm.com> > > v6-v5:Fix pep8, delete migratation finish func. > Signed-off-by: Simon Jin<simonjin(a)linux.vnet.ibm.com> > --- > docs/API.md | 5 ++ > src/kimchi/control/vms.py | 3 + > src/kimchi/i18n.py | 8 +++ > src/kimchi/model/vms.py | 166 +++++++++++++++++++++++++++++++++++++++++++++- > 4 files changed, 180 insertions(+), 2 deletions(-) > > diff --git a/docs/API.md b/docs/API.md > index 210a036..4408764 100644 > --- a/docs/API.md > +++ b/docs/API.md > @@ -132,6 +132,11 @@ the following general conventions: > It emulates the power reset button on a machine. Note that there is a > risk of data loss caused by reset without the guest OS shutdown. > * connect: Prepare the connection for spice or vnc > +* migrate: Migrate a virtual machine to a remote server, only support live mode without block migration. > + * remoteHost: IP address or hostname of the remote server. > + * user: User of the remote server. > + * passwd: Passwd of user on remote server. > + * secure: Migrate in P2P secure tunel mode.(optional) Can we add this func to API.json so that parameters can be checked?

> * clone: Create a new VM identical to this VM. The new VM's name, UUID and > network MAC addresses will be generated automatically. Each existing > diff --git a/src/kimchi/control/vms.py b/src/kimchi/control/vms.py > index a1589ef..3638b3a 100644 > --- a/src/kimchi/control/vms.py > +++ b/src/kimchi/control/vms.py > @@ -42,6 +42,9 @@ class VM(Resource): > for ident, node in sub_nodes.items(): > setattr(self, ident, node(model, self.ident)) > self.start = self.generate_action_handler('start') > + self.migrate = self.generate_action_handler_task('migrate', > + ['remoteHost', 'user', > + 'passwd', 'secure']) > self.poweroff = self.generate_action_handler('poweroff') > self.shutdown = self.generate_action_handler('shutdown') > self.reset = self.generate_action_handler('reset') > diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py > index e3a051c..81a6dbb 100644 > --- a/src/kimchi/i18n.py > +++ b/src/kimchi/i18n.py > @@ -107,6 +107,14 @@ messages = { > "KCHVM0033E": _("Virtual machine '%(name)s' must be stopped before cloning it."), > "KCHVM0034E": _("Insufficient disk space to clone virtual machine '%(name)s'"), > "KCHVM0035E": _("Unable to clone VM '%(name)s'. Details: %(err)s"), > + "KCHVM0036E": _("Migrate %(name)s to localhost %(remoteHost)s is not allowed."), > + "KCHVM0037E": _("vm %(name)s is already in migrating, can not perform migrate on it again."), > + "KCHVM0038E": _("Can't migrate vm %(name)s on Hypervisor arch %(srcarch)s.to a different Hypervisor arch %(destarch)s."), > + "KCHVM0039E": _("Can't migrate %(name)s from %(srchyp)s to a different Hypervisor type %(desthyp)s."), > + "KCHVM0040E": _("Can't migrate vm %(name)s, vm has passthrough device %(hostdevs)s."), > + "KCHVM0041E": _("Can not migrate vm %(name)s when its in %(state)s state."), > + "KCHVM0042E": _("Failed Migrate vm %(name)s due error: %(err)s"), > + "KCHVM0043E": _("Failed to destroy vm %(name)s from migration recover. Details: %(err)s"), > > "KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), > "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), > diff --git a/src/kimchi/model/vms.py b/src/kimchi/model/vms.py > index 3aa1145..023bd10 100644 > --- a/src/kimchi/model/vms.py > +++ b/src/kimchi/model/vms.py > @@ -33,9 +33,10 @@ from cherrypy.process.plugins import BackgroundTask > from kimchi import model, vnc > from kimchi.config import READONLY_POOL_TYPE, config > from kimchi.exception import InvalidOperation, InvalidParameter > -from kimchi.exception import NotFoundError, OperationFailed > +from kimchi.exception import NotFoundError, OperationFailed, KimchiException Why do we need to introduce general error KimchiException?

> from kimchi.model.config import CapabilitiesModel > -from kimchi.model.tasks import TaskModel > +from kimchi.model.libvirtconnection import LibvirtConnection > +from kimchi.model.tasks import TaskModel, TasksModel > from kimchi.model.templates import TemplateModel > from kimchi.model.utils import get_vm_name > from kimchi.model.utils import get_metadata_node > @@ -268,6 +269,7 @@ class VMModel(object): > self.groups = import_class('kimchi.model.groups.GroupsModel')(**kargs) > self.vms = VMsModel(**kargs) > self.task = TaskModel(**kargs) > + self.tasks = TasksModel(**kargs) > self.storagepool = model.storagepools.StoragePoolModel(**kargs) > self.storagevolume = model.storagevolumes.StorageVolumeModel(**kargs) > self.storagevolumes = model.storagevolumes.StorageVolumesModel(**kargs) > @@ -850,6 +852,166 @@ class VMModel(object): > raise OperationFailed("KCHVM0019E", > {'name': name, 'err': e.get_error_message()}) > > + def _preCheck(self, name, destConn, remoteHost): > + """Pre check before migration""" > + > + kimchi_log.debug('precheck migrate %s' % name) > + _destConn = destConn.get() > + if remoteHost in ['localhost', '127.0.0.1']: > + kimchi_log.debug('vm %s Can not migrate to localhost.' % name) > + raise OperationFailed("KCHVM0036E", {'name': name, > + 'remoteHost': remoteHost}) > + # check if the vm already in migration > + try: > + alltasks = self.tasks.get_list() > + for taskid in alltasks: > + taskobj = self.task.lookup(taskid) > + if taskobj['target_uri'] == "/vms/%s/migrate" % name \ > + and taskobj['status'] == "running": We need to abstract a helper function for this because we use target_uri filtering a lot.

Besides, I think we need a lock here rather than just check the task id, because we enter this part, we are ahead of the actual creation of tasks. we may have 2 migrations pass this check before both running to "add_task" line.

Or do we have a libvirt/qemu lock to leverage so that we can probe its status?

> + kimchi_log.debug('vm %s is already in migrating.' % name) > + raise OperationFailed("KCHVM0037E", {'name': name}) > + except NotFoundError: > + kimchi_log.debug('No previous migrate on vm %s.' % name) > + > + srcarch = self.conn.get().getInfo()[0] > + destarch = _destConn.getInfo()[0] > + if srcarch != destarch: > + kimchi_log.debug('vm %s can not migrate to different arch server.' > + % (name, destarch)) > + raise OperationFailed("KCHVM0038E", {'name': name, > + 'srcarch': srcarch, > + 'destarch': destarch}) AFAIK, these requirements seem not enough to guarantee a migration,(see http://wiki.libvirt.org/page/VM_lifecycle#Migration) Requirements for migration: * Shared storage accessible under same paths and locations,e.g. iSCSI, NFS, etc. * Exactly the same versions of hypervisor software on both physical hosts * Same network configuration. * Same CPUs or better. The CPUs must be from the same vendor and the CPU flags on the destination host must be superset of CPU flags on the source host.

> + srchyp = self.conn.get().getType() > + desthyp = _destConn.getType() > + if srchyp != desthyp: > + kimchi_log.debug('vm %s can not migrate to a different hypervisor' > + % name) > + raise OperationFailed("KCHVM0039E", {'name': name, > + 'srchyp': srchyp, > + 'desthyp': desthyp}) > + > + # model.host import DOM_STATE_MAP, vmhostdevs import host, > + # to void loop import DOM_STATE_MAP, move import VMHostDevsModel here > + from kimchi.model.vmhostdevs import VMHostDevsModel > + # check if there is any passthrough devices belong to this vm > + vmhostdevs = VMHostDevsModel(conn=self.conn) > + hostdevs = vmhostdevs.get_list(name) > + if hostdevs: > + raise OperationFailed("KCHVM0040E", {'name': name, > + 'hostdevs': hostdevs}) > + > + @staticmethod > + def _getRemoteConn(remoteHost, user, passwd, transport='ssh'): > + """open destination connect to migrate""" > + > + duri = 'qemu+%s://%s@%s/system' % (transport, user, remoteHost) > + conn = LibvirtConnection(duri) > + return conn > + > + def _do_migrate(self, cb, params): > + """Asynchronous function which performs the migrate operation. > + > + Arguments: > + cb -- A callback function to signal the Task's progress. > + params -- A dict with the following values: > + "name": Name of the virtual machine to be migrated. > + "destConn": A LibvirtConnection to destination server. > + "remoteHost": The migration destination server. > + "secure": Migrate in P2P secure tunel mode.(optional) > + """ > + > + name = params['name'].decode('utf-8') > + destConn = params['destConn'] > + remoteHost = params['remoteHost'] > + secure = params['secure'] > + _destConn = destConn.get() We can put the destConn.get() to function to _getRemoteConn to avoid duplicate code.

> + > + cb('starting a migration') > + kimchi_log.debug('migrate %s start' % name) > + dom = self.get_vm(name, self.conn) > + flag = 0 > + if secure: > + flag |= (libvirt.VIR_MIGRATE_PEER2PEER | > + libvirt.VIR_MIGRATE_TUNNELLED) > + > + state = DOM_STATE_MAP[dom.info()[0]] > + if state in ['running', 'paused']: > + flag |= libvirt.VIR_MIGRATE_LIVE > + elif state == 'shutoff': > + flag |= (libvirt.VIR_MIGRATE_OFFLINE | > + libvirt.VIR_MIGRATE_PERSIST_DEST) > + else: > + kimchi_log.debug('Can not migrate vm %s when its in %s.' > + % (name, state)) > + raise OperationFailed("KCHVM0041E", {'name': name, > + 'state': state}) > + try: > + dom.migrate(_destConn, flag, None, None, 0) > + except libvirt.libvirtError as e: > + kimchi_log.error('migrate %s to %s failed' % (name, remoteHost)) > + self._recover(name, destConn, cb) > + kimchi_log.debug('finish migrate %s' % name) > + cb('Migrate finished', False) > + raise OperationFailed('KCHVM0042E', {'err': e.message, > + 'name': name}) > + kimchi_log.debug('finish migrate %s' % name) > + cb('Migrate finished', True) > + > + def migrate(self, name, remoteHost, user, passwd=None, secure=False): > + """Migrate a virtual machine to a remote server. > + > + Arguments: > + name -- The name of the virtual machine to be migrated. > + remoteHost -- Remote server that the VM will be migrated to. > + user -- Login user on remote server,prefer root. > + passwd -- The password of the user on remote server, > + if the local server > + has ssk key exchanged, then this param can be NULL. > + secure -- Migrate in P2P secure tunel mode.(optional) > + > + Return: > + A Task running the clone operation. > + """ > + name = name.decode('utf-8') > + remoteHost = remoteHost.decode('utf-8') > + user = user.decode('utf-8') > + > + destConn = self._getRemoteConn(remoteHost, user, passwd) > + self._preCheck(name, destConn, remoteHost) > + > + params = {'name': name, > + 'destConn': destConn, > + 'remoteHost': remoteHost, > + 'secure': secure} > + task_id = add_task('/vms/%s/migrate' % name, self._do_migrate, > + self.objstore, params) > + > + return self.task.lookup(task_id) > + > + def _recover(self, name, destconn, cb): > + """Recover from a failed migration. > + > + Arguments: > + "cb": A callback function to signal the Task's progress. > + "name": Name of the virtual machine to be migrated. > + "destConn": A LibvirtConnection to destination server. > + """ > + try: > + destvm = self.get_vm(name, destconn) > + except KimchiException as e: > + if type(e) == NotFoundError: > + return > + > + cb('recover from a failed migration', False) > + kimchi_log.debug('recover from a failed migrate %s' % name) > + try: > + destvm.destroy() > + kimchi_log.error("Failed to destory migrate vm %s on \ > + destination server" % name) > + except libvirt.libvirtError as e: > + raise OperationFailed('KCHVM0043E', {'name': name, > + 'err': e.message}) > + > def poweroff(self, name): > dom = self.get_vm(name, self.conn) > try: