Re: [Kimchi-devel] [PATCH v2] Live migration backend: non-shared storage VM migration

From: Daniel Henrique Barboza <dhbarboza82(a)gmail.com&gt; This patch implements non-shared storage VM migration in the existing live migration backend. It was necessary to complement libvirt support by creating any disk or ISO prior to the migration process in the remote destination host. Signed-off-by: Daniel Henrique Barboza <dhbarboza82(a)gmail.com&gt; --- src/wok/plugins/kimchi/i18n.py | 4 + src/wok/plugins/kimchi/model/vms.py | 104 +++++++++++++++++++- src/wok/plugins/kimchi/tests/test_livemigration.py | 107 ++++++++++++++++----- 3 files changed, 189 insertions(+), 26 deletions(-) diff --git a/src/wok/plugins/kimchi/i18n.py b/src/wok/plugins/kimchi/i18n.py index 59b61de..5e9eee4 100644 --- a/src/wok/plugins/kimchi/i18n.py +++ b/src/wok/plugins/kimchi/i18n.py @@ -115,6 +115,10 @@ messages = { "KCHVM0058E": _("Failed to migrate virtual machine %(name)s due error: %(err)s"), "KCHVM0059E": _("User name of the remote server must be a string."), "KCHVM0060E": _("Destination host of the migration must be a string."), + "KCHVM0061E": _("Unable to create file %(path)s at %(host)s using user %(user)s."), + "KCHVM0062E": _("Unable to read disk size of %(path)s, error: %(error)s"), + "KCHVM0063E": _("Unable to create disk image %(path)s at %(host)s using user %(user)s. Error: %(error)s"), + "KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), diff --git a/src/wok/plugins/kimchi/model/vms.py b/src/wok/plugins/kimchi/model/vms.py index f1ae8c2..c98558c 100644 --- a/src/wok/plugins/kimchi/model/vms.py +++ b/src/wok/plugins/kimchi/model/vms.py @@ -55,6 +55,7 @@ from wok.plugins.kimchi.model.utils import set_metadata_node from wok.plugins.kimchi.screenshot import VMScreenshot from wok.plugins.kimchi.utils import template_name_from_uri from wok.plugins.kimchi.xmlutils.cpu import get_cpu_xml, get_numa_xml +from wok.plugins.kimchi.xmlutils.disk import get_vm_disk_info, get_vm_disks DOM_STATE_MAP = {0: 'nostate', @@ -1342,6 +1343,87 @@ class VMModel(object): self._check_if_host_not_localhost(remote_host) self._check_if_password_less_login_enabled(remote_host, user) + def _check_if_path_exists_in_remote_host(self, path, remote_host, user): + username_host = "%s@%s" % (user, remote_host) + cmd = ['ssh', '-oStrictHostKeyChecking=no', username_host, + 'test', '-f', path] + _, _, returncode = run_command(cmd, 5, silent=True) + return returncode == 0 + + def _get_vm_devices_infos(self, vm_name): + dom = VMModel.get_vm(vm_name, self.conn) + infos = [get_vm_disk_info(dom, dev_name) + for dev_name in get_vm_disks(dom).keys()] + return infos + + def _check_if_nonshared_migration(self, vm_name, remote_host, user): + for dev_info in self._get_vm_devices_infos(vm_name): + dev_path = dev_info.get('path') + if not self._check_if_path_exists_in_remote_host( + dev_path, remote_host, user): + return True + return False + + def _create_remote_path(self, path, remote_host, user): + username_host = "%s@%s" % (user, remote_host) + cmd = ['ssh', '-oStrictHostKeyChecking=no', username_host, + 'touch', path] + _, _, returncode = run_command(cmd, 5, silent=True) + if returncode != 0: + raise OperationFailed( + "KCHVM0061E", + {'path': path, 'host': remote_host, 'user': user} + ) + + def _get_img_size(self, disk_path): + try: + conn = self.conn.get() + vol_obj = conn.storageVolLookupByPath(disk_path) + return vol_obj.info()[1] + except Exception, e: + raise OperationFailed( + "KCHVM0062E", + {'path': disk_path, 'error': e.message} + ) + + def _create_remote_disk(self, disk_info, remote_host, user): + username_host = "%s@%s" % (user, remote_host) + disk_fmt = disk_info.get('format') + disk_path = disk_info.get('path') + disk_size = self._get_img_size(disk_path) + cmd = ['ssh', '-oStrictHostKeyChecking=no', username_host, + 'qemu-img', 'create', '-f', disk_fmt, + disk_path, str(disk_size)] + out, err, returncode = run_command(cmd, silent=True) + if returncode != 0: + raise OperationFailed( + "KCHVM0063E", + { + 'error': err, + 'path': disk_path, + 'host': remote_host, + 'user': user + } + ) + + def _create_vm_remote_paths(self, vm_name, remote_host, user): + for dev_info in self._get_vm_devices_infos(vm_name): + dev_path = dev_info.get('path') + if not self._check_if_path_exists_in_remote_host( + dev_path, remote_host, user): + if dev_info.get('type') == 'cdrom': + self._create_remote_path( + dev_path, + remote_host, + user + ) + else: + self._create_remote_disk( + dev_info, + remote_host, + user + ) + def migrate(self, name, remote_host, user='root'): name = name.decode('utf-8') remote_host = remote_host.decode('utf-8') @@ -1349,8 +1431,17 @@ class VMModel(object): self.migration_pre_check(remote_host, user) dest_conn = self._get_remote_libvirt_conn(remote_host) + non_shared = self._check_if_nonshared_migration( + name, + remote_host, + user + ) + params = {'name': name, - 'dest_conn': dest_conn} + 'dest_conn': dest_conn, + 'non_shared': non_shared, + 'remote_host': remote_host, + 'user': user} task_id = add_task('/vms/%s/migrate' % name, self._migrate_task, self.objstore, params) @@ -1359,6 +1450,9 @@ class VMModel(object): def _migrate_task(self, cb, params): name = params['name'].decode('utf-8') dest_conn = params['dest_conn'] + non_shared = params['non_shared'] + remote_host = params['remote_host'] + user = params['user'] cb('starting a migration') @@ -1377,6 +1471,14 @@ class VMModel(object): dest_conn.close() raise OperationFailed("KCHVM0057E", {'name': name, 'state': state}) + if non_shared: + flags |= libvirt.VIR_MIGRATE_NON_SHARED_DISK + self._create_vm_remote_paths( + name, + remote_host, + user + ) + try: dom.migrate(dest_conn, flags) except libvirt.libvirtError as e: diff --git a/src/wok/plugins/kimchi/tests/test_livemigration.py b/src/wok/plugins/kimchi/tests/test_livemigration.py index 1fdcd65..b9eb9a4 100644 --- a/src/wok/plugins/kimchi/tests/test_livemigration.py +++ b/src/wok/plugins/kimchi/tests/test_livemigration.py @@ -21,7 +21,6 @@ import json import libvirt import os import socket -import shutil import unittest from functools import partial @@ -29,6 +28,7 @@ from functools import partial from wok.basemodel import Singleton from wok.exception import OperationFailed from wok.rollbackcontext import RollbackContext +from wok.utils import run_command from wok.plugins.kimchi.model import model @@ -42,14 +42,14 @@ from utils import get_free_port, patch_auth, request from utils import run_server, wait_task -TMP_DIR = '/var/lib/kimchi/tests/' -UBUNTU_ISO = TMP_DIR + 'ubuntu14.04.iso' +ISO_DIR = '/var/lib/libvirt/images/' +UBUNTU_ISO = ISO_DIR + 'ubuntu_kimchi_migration_test_14.04.iso' KIMCHI_LIVE_MIGRATION_TEST = None def setUpModule(): - if not os.path.exists(TMP_DIR): - os.makedirs(TMP_DIR) + if not os.path.exists(ISO_DIR): + os.makedirs(ISO_DIR) iso_gen.construct_fake_iso(UBUNTU_ISO, True, '14.04', 'ubuntu') # Some FeatureTests functions depend on server to validate their result. # As CapabilitiesModel is a Singleton class it will get the first result @@ -61,7 +61,7 @@ def setUpModule(): def tearDownModule(): - shutil.rmtree(TMP_DIR) + os.remove(UBUNTU_ISO) def remoteserver_environment_defined(): @@ -89,24 +89,40 @@ def check_if_vm_migration_test_possible(): class LiveMigrationTests(unittest.TestCase): def setUp(self): self.tmp_store = '/tmp/kimchi-store-test' - self.inst = model.Model(objstore_loc=self.tmp_store) + self.inst = model.Model( + 'qemu:///system', + objstore_loc=self.tmp_store + ) params = {'name': u'template_test_vm_migrate', 'disks': [], 'cdrom': UBUNTU_ISO, 'memory': 2048, 'max_memory': 4096*1024} self.inst.templates_create(params) + params = {'name': u'template_test_vm_migrate_nonshared', + 'disks': [{'name': 'test_vm_migrate.img', 'size': 1}], + 'cdrom': UBUNTU_ISO, + 'memory': 2048, + 'max_memory': 4096*1024} + self.inst.templates_create(params) def tearDown(self): self.inst.template_delete('template_test_vm_migrate') + self.inst.template_delete('template_test_vm_migrate_nonshared') os.unlink(self.tmp_store) - def create_vm_test(self): + def create_vm_test(self, non_shared_storage=False): params = { 'name': u'test_vm_migrate', 'template': u'/plugins/kimchi/templates/template_test_vm_migrate' } + if non_shared_storage: + params = { + 'name': u'test_vm_migrate', + 'template': u'/plugins/kimchi/templates/' + 'template_test_vm_migrate_nonshared' + } task = self.inst.vms_create(params) self.inst.task_wait(task['id']) @@ -171,8 +187,6 @@ class LiveMigrationTests(unittest.TestCase): @unittest.skipUnless(check_if_vm_migration_test_possible(), 'not possible to test a live migration') def test_vm_livemigrate_persistent(self): - inst = model.Model(libvirt_uri='qemu:///system', - objstore_loc=self.tmp_store) with RollbackContext() as rollback: self.create_vm_test() @@ -189,9 +203,9 @@ class LiveMigrationTests(unittest.TestCase): except Exception, e: self.fail('Failed to start the vm, reason: %s' % e.message) try: - task = inst.vm_migrate('test_vm_migrate', - KIMCHI_LIVE_MIGRATION_TEST) - inst.task_wait(task['id']) + task = self.inst.vm_migrate('test_vm_migrate', + KIMCHI_LIVE_MIGRATION_TEST) + self.inst.task_wait(task['id']) self.assertIn('test_vm_migrate', self.get_remote_vm_list()) remote_conn = self.get_remote_conn() @@ -208,9 +222,6 @@ class LiveMigrationTests(unittest.TestCase): @unittest.skipUnless(check_if_vm_migration_test_possible(), 'not possible to test a live migration') def test_vm_livemigrate_transient(self): - inst = model.Model(libvirt_uri='qemu:///system', - objstore_loc=self.tmp_store) - self.create_vm_test() with RollbackContext() as rollback: @@ -229,9 +240,9 @@ class LiveMigrationTests(unittest.TestCase): ) vm.undefine() - task = inst.vm_migrate('test_vm_migrate', - KIMCHI_LIVE_MIGRATION_TEST) - inst.task_wait(task['id']) + task = self.inst.vm_migrate('test_vm_migrate', + KIMCHI_LIVE_MIGRATION_TEST) + self.inst.task_wait(task['id']) self.assertIn('test_vm_migrate', self.get_remote_vm_list()) remote_conn = self.get_remote_conn() @@ -258,9 +269,6 @@ class LiveMigrationTests(unittest.TestCase): @unittest.skipUnless(check_if_vm_migration_test_possible(), 'not possible to test shutdown migration') def test_vm_coldmigrate(self): - inst = model.Model(libvirt_uri='qemu:///system', - objstore_loc=self.tmp_store) - with RollbackContext() as rollback: self.create_vm_test() rollback.prependDefer(utils.rollback_wrapper, self.inst.vm_delete, @@ -272,9 +280,9 @@ class LiveMigrationTests(unittest.TestCase): self.inst.vmstorage_delete('test_vm_migrate', dev_list[0]) try: - task = inst.vm_migrate('test_vm_migrate', - KIMCHI_LIVE_MIGRATION_TEST) - inst.task_wait(task['id']) + task = self.inst.vm_migrate('test_vm_migrate', + KIMCHI_LIVE_MIGRATION_TEST) + self.inst.task_wait(task['id']) self.assertIn('test_vm_migrate', self.get_remote_vm_list()) remote_conn = self.get_remote_conn() @@ -290,6 +298,55 @@ class LiveMigrationTests(unittest.TestCase): except Exception, e: self.fail('Migration test failed: %s' % e.message) + def _erase_remote_file(self, path): + username_host = "root@%s" % KIMCHI_LIVE_MIGRATION_TEST + cmd = ['ssh', '-oStrictHostKeyChecking=no', username_host, + 'rm', '-f', path] + _, _, returncode = run_command(cmd, silent=True) + if returncode != 0: + print 'cannot erase remote file ', path + + @unittest.skipUnless(check_if_vm_migration_test_possible(), + 'not possible to test a live migration') + def test_vm_livemigrate_persistent_nonshared(self): + + with RollbackContext() as rollback: + self.create_vm_test(non_shared_storage=True) + rollback.prependDefer(utils.rollback_wrapper, self.inst.vm_delete, + u'test_vm_migrate') + + # getting disk path info to clean it up later + storage_list = self.inst.vmstorages_get_list('test_vm_migrate') + disk_info = self.inst.vmstorage_lookup( + 'test_vm_migrate', + storage_list[0] + ) + disk_path = disk_info.get('path') + + try: + self.inst.vm_start('test_vm_migrate') + except Exception, e: + self.fail('Failed to start the vm, reason: %s' % e.message) + try: + task = self.inst.vm_migrate('test_vm_migrate', + KIMCHI_LIVE_MIGRATION_TEST) + self.inst.task_wait(task['id'], 3600) + self.assertIn('test_vm_migrate', self.get_remote_vm_list()) + + remote_conn = self.get_remote_conn() + rollback.prependDefer(remote_conn.close) + + remote_vm = remote_conn.lookupByName('test_vm_migrate') + self.assertTrue(remote_vm.isPersistent()) + + remote_vm.destroy() + remote_vm.undefine() + + self._erase_remote_file(disk_path) + self._erase_remote_file(UBUNTU_ISO) + except Exception, e: + self.fail('Migration test failed: %s' % e.message) + def _task_lookup(self, taskid): return json.loads( self.request('/plugins/kimchi/tasks/%s' % taskid).read()