[Kimchi-devel] [PATCH] Create logical pool from existing VG

Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com&gt; Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com&gt; --- src/wok/plugins/kimchi/API.json | 5 + src/wok/plugins/kimchi/control/host.py | 22 ++++ src/wok/plugins/kimchi/disks.py | 127 ++++++++++++++++++--- src/wok/plugins/kimchi/docs/API.md | 25 +++- src/wok/plugins/kimchi/i18n.py | 5 + src/wok/plugins/kimchi/mockmodel.py | 27 +++++ src/wok/plugins/kimchi/model/host.py | 27 +++++ src/wok/plugins/kimchi/model/libvirtstoragepool.py | 9 +- src/wok/plugins/kimchi/model/storagepools.py | 30 +++-- .../plugins/kimchi/tests/test_mock_storagepool.py | 8 +- 10 files changed, 255 insertions(+), 30 deletions(-) diff --git a/src/wok/plugins/kimchi/API.json b/src/wok/plugins/kimchi/API.json index e236e51..ab9ac9b 100644 --- a/src/wok/plugins/kimchi/API.json +++ b/src/wok/plugins/kimchi/API.json @@ -141,6 +141,11 @@ "error": "KCHPOOL0025E" } } + }, + "from_vg": { + "description": "Indicate if a logical pool will be created from an existing VG or not", + "type": "boolean", + "error": "KCHPOOL0026E" } } } diff --git a/src/wok/plugins/kimchi/control/host.py b/src/wok/plugins/kimchi/control/host.py index 9bc703a..39df0d0 100644 --- a/src/wok/plugins/kimchi/control/host.py +++ b/src/wok/plugins/kimchi/control/host.py @@ -35,12 +35,34 @@ class Host(Resource): self.devices = Devices(self.model) self.cpuinfo = CPUInfo(self.model) self.partitions = Partitions(self.model) + self.vgs = VolumeGroups(self.model) @property def data(self): return {} +class VolumeGroups(Collection): + def __init__(self, model): + super(VolumeGroups, self).__init__(model) + self.role_key = 'host' + self.uri_fmt = "/host/vgs" + self.admin_methods = ['GET'] + self.resource = VolumeGroup + + +class VolumeGroup(Resource): + def __init__(self, model, id=None): + super(VolumeGroup, self).__init__(model, id) + self.role_key = 'host' + self.uri_fmt = "/host/vgs/%s" + self.admin_methods = ['GET'] + + @property + def data(self): + return self.info + + class VMHolders(SimpleCollection): def __init__(self, model, device_id): super(VMHolders, self).__init__(model) diff --git a/src/wok/plugins/kimchi/disks.py b/src/wok/plugins/kimchi/disks.py index eb40e3a..5515026 100644 --- a/src/wok/plugins/kimchi/disks.py +++ b/src/wok/plugins/kimchi/disks.py @@ -19,12 +19,11 @@ import os.path import re -import subprocess from parted import Device as PDevice from parted import Disk as PDisk from wok.exception import OperationFailed -from wok.utils import wok_log +from wok.utils import run_command, wok_log def _get_dev_node_path(maj_min): @@ -46,11 +45,8 @@ def _get_dev_node_path(maj_min): def _get_lsblk_devs(keys, devs=[]): - lsblk = subprocess.Popen( - ["lsblk", "-Pbo"] + [','.join(keys)] + devs, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = lsblk.communicate() - if lsblk.returncode != 0: + out, err, rc = run_command(["lsblk", "-Pbo"] + [','.join(keys)] + devs) + if rc != 0: raise OperationFailed("KCHDISKS0001E", {'err': err}) return _parse_lsblk_output(out, keys) @@ -127,12 +123,9 @@ def _parse_lsblk_output(output, keys): def _get_vgname(devNodePath): """ Return volume group name of a physical volume. If the device node path is not a physical volume, return empty string. """ - pvs = subprocess.Popen( - ["pvs", "--unbuffered", "--nameprefixes", "--noheadings", - "-o", "vg_name", devNodePath], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = pvs.communicate() - if pvs.returncode != 0: + out, err, rc = run_command(["pvs", "--unbuffered", "--nameprefixes", + "--noheadings", "-o", "vg_name", devNodePath]) + if rc != 0: return "" return re.findall(r"LVM2_VG_NAME='([^\']*)'", out)[0] @@ -194,3 +187,111 @@ def get_partition_details(name): dev['path'] = dev_path dev['name'] = name return dev + + +def vgs(): + """ + lists all volume groups in the system. All size units are in bytes. + + [{'vgname': 'vgtest', 'size': 999653638144L, 'free': 0}] + """ + cmd = ['vgs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'vg_name,vg_size,vg_free'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHLVM0001E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of VGs + vgs = map(lambda v: v.strip(), out.strip('\n').split('\n')) + + # create a dict based on data retrieved from vgs + return map(lambda l: {'vgname': l[0], + 'size': long(l[1]), + 'free': long(l[2])}, + [fields.split() for fields in vgs]) + + +def lvs(vgname=None): + """ + lists all logical volumes found in the system. It can be filtered by + the volume group. All size units are in bytes. + + [{'lvname': 'lva', 'path': '/dev/vgtest/lva', 'size': 12345L}, + {'lvname': 'lvb', 'path': '/dev/vgtest/lvb', 'size': 12345L}] + """ + cmd = ['lvs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'lv_name,lv_path,lv_size,vg_name'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHLVM0001E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of LVs filtered by vgname, if + # provided + lvs = filter(lambda f: vgname is None or vgname in f, + map(lambda v: v.strip(), out.strip('\n').split('\n'))) + + # create a dict based on data retrieved from lvs + return map(lambda l: {'lvname': l[0], + 'path': l[1], + 'size': long(l[2])}, + [fields.split() for fields in lvs]) + + +def pvs(vgname=None): + """ + lists all physical volumes in the system. It can be filtered by the + volume group. All size units are in bytes. + + [{'pvname': '/dev/sda3', + 'size': 469502001152L, + 'uuid': 'kkon5B-vnFI-eKHn-I5cG-Hj0C-uGx0-xqZrXI'}, + {'pvname': '/dev/sda2', + 'size': 21470642176L, + 'uuid': 'CyBzhK-cQFl-gWqr-fyWC-A50Y-LMxu-iHiJq4'}] + """ + cmd = ['pvs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'pv_name,pv_size,pv_uuid,vg_name'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHLVM0001E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of PVs filtered by vgname, if + # provided + pvs = filter(lambda f: vgname is None or vgname in f, + map(lambda v: v.strip(), out.strip('\n').split('\n'))) + + # create a dict based on data retrieved from pvs + return map(lambda l: {'pvname': l[0], + 'size': long(l[1]), + 'uuid': l[2]}, + [fields.split() for fields in pvs]) diff --git a/src/wok/plugins/kimchi/docs/API.md b/src/wok/plugins/kimchi/docs/API.md index 5f0b234..5787755 100644 --- a/src/wok/plugins/kimchi/docs/API.md +++ b/src/wok/plugins/kimchi/docs/API.md @@ -445,7 +445,8 @@ A interface represents available network interface on VM. * **GET**: Retrieve a summarized list of all defined Storage Pools * **POST**: Create a new Storage Pool - * name: The name of the Storage Pool. + * name: The name of the Storage Pool. It corresponds to the VG name while + creating a logical storage pool from an existing VG. * type: The type of the defined Storage Pool. Supported types: 'dir', 'kimchi-iso', 'netfs', 'logical', 'iscsi', 'scsi' * path: The path of the defined Storage Pool. @@ -467,6 +468,8 @@ A interface represents available network interface on VM. * username: Login username of the iSCSI target. * password: Login password of the iSCSI target. * adapter_name: SCSI host name. + * from_vg: Indicate if a logical pool will be created from an + existing VG or not. ### Resource: Storage Pool @@ -898,6 +901,26 @@ List of available groups. Otherwise blank. * available: false, if the partition is in use by system; true, otherwise. +### Collection: Volume Groups + +**URI:** /plugins/kimchi/host/vgs + +**Methods:** + +* **GET**: Retrieves a detailed list of all volume groups in the host. + +### Resource: Volume Group + +**URI:** /plugins/kimchi/host/vgs/*:name* + +**Methods:** + +* **GET**: Retrieve the description of a single Volume Group: + * name: The volume group name. Used to identify it in this API. + * lvs: The logical volumes associated to this volume group. + * pvs: The phisical volumes associated to this volume group. + * free: Amount of free space in the volume group. + * size: Total size of the volume group. ### Collection: Peers diff --git a/src/wok/plugins/kimchi/i18n.py b/src/wok/plugins/kimchi/i18n.py index c69d072..aede103 100644 --- a/src/wok/plugins/kimchi/i18n.py +++ b/src/wok/plugins/kimchi/i18n.py @@ -30,6 +30,8 @@ messages = { "KCHDISKS0001E": _("Error while getting block devices. Details: %(err)s"), "KCHDISKS0002E": _("Error while getting block device information for %(device)s."), + "KCHLVM0001E": _("Unable to retrieve LVM information. Details: %(err)s"), + "KCHPART0001E": _("Partition %(name)s does not exist in the host"), "KCHDEVS0001E": _('Unknown "_cap" specified'), @@ -210,6 +212,7 @@ messages = { "KCHPOOL0035E": _("Unable to delete pool %(name)s as it is associated with some templates"), "KCHPOOL0036E": _("A volume group named '%(name)s' already exists. Please, choose another name to create the logical pool."), "KCHPOOL0037E": _("Unable to update database with deep scan information due error: %(err)s"), + "KCHPOOL0038E": _("No volume group '%(name)s' found. Please, specify an existing volume group to create the logical pool from."), "KCHVOL0001E": _("Storage volume %(name)s already exists"), "KCHVOL0002E": _("Storage volume %(name)s does not exist in storage pool %(pool)s"), @@ -305,4 +308,6 @@ messages = { "KCHCPUINF0002E": _("Invalid vCPU/topology combination."), "KCHCPUINF0003E": _("This host (or current configuration) does not allow CPU topology."), + "KCHLVMS0001E": _("Invalid volume group name parameter: %(name)s."), + } diff --git a/src/wok/plugins/kimchi/mockmodel.py b/src/wok/plugins/kimchi/mockmodel.py index 895401e..9186f78 100644 --- a/src/wok/plugins/kimchi/mockmodel.py +++ b/src/wok/plugins/kimchi/mockmodel.py @@ -39,6 +39,7 @@ from wok.plugins.kimchi.model.libvirtstoragepool import NetfsPoolDef from wok.plugins.kimchi.model.libvirtstoragepool import StoragePoolDef from wok.plugins.kimchi.model.model import Model from wok.plugins.kimchi.model.storagepools import StoragePoolModel +from wok.plugins.kimchi.model.storagepools import StoragePoolsModel from wok.plugins.kimchi.model.storagevolumes import StorageVolumeModel from wok.plugins.kimchi.model.storagevolumes import StorageVolumesModel from wok.plugins.kimchi.model import storagevolumes @@ -73,6 +74,7 @@ class MockModel(Model): defaults.update(mockmodel_defaults) osinfo.defaults = dict(defaults) + self._mock_vgs = MockVolumeGroups() self._mock_partitions = MockPartitions() self._mock_devices = MockDevices() self._mock_storagevolumes = MockStorageVolumes() @@ -113,6 +115,7 @@ class MockModel(Model): setattr(self, m, mock_method) DeviceModel.lookup = self._mock_device_lookup + StoragePoolsModel._check_lvm = self._check_lvm StoragePoolModel._update_lvm_disks = self._update_lvm_disks StorageVolumesModel.get_list = self._mock_storagevolumes_get_list StorageVolumeModel.doUpload = self._mock_storagevolume_doUpload @@ -252,6 +255,10 @@ class MockModel(Model): return MockModel._libvirt_get_vol_path(pool, vol) + def _check_lvm(self, name, from_vg): + # do not do any verification while using MockModel + pass + def _update_lvm_disks(self, pool_name, disks): conn = self.conn.get() pool = conn.storagePoolLookupByName(pool_name.encode('utf-8')) @@ -334,6 +341,12 @@ class MockModel(Model): def _mock_partition_lookup(self, name): return self._mock_partitions.partitions[name] + def _mock_volumegroups_get_list(self): + return self._mock_vgs.data.keys() + + def _mock_volumegroup_lookup(self, name): + return self._mock_vgs.data[name] + def _mock_vm_clone(self, name): new_name = get_next_clone_name(self.vms_get_list(), name) snapshots = MockModel._mock_snapshots.get(name, []) @@ -418,6 +431,20 @@ class MockStorageVolumes(object): 'isvalid': True}} +class MockVolumeGroups(object): + def __init__(self): + self.data = {"hostVG": {"lvs": [], + "name": "hostVG", + "pvs": ["/dev/vdx"], + "free": 5347737600, + "size": 5347737600}, + "kimchiVG": {"lvs": [], + "name": "kimchiVG", + "pvs": ["/dev/vdz", "/dev/vdw"], + "free": 10695475200, + "size": 10695475200}} + + class MockPartitions(object): def __init__(self): self.partitions = {"vdx": {"available": True, "name": "vdx", diff --git a/src/wok/plugins/kimchi/model/host.py b/src/wok/plugins/kimchi/model/host.py index 96f4bea..f3ca946 100644 --- a/src/wok/plugins/kimchi/model/host.py +++ b/src/wok/plugins/kimchi/model/host.py @@ -241,3 +241,30 @@ class PartitionModel(object): def lookup(self, name): return disks.get_partition_details(name) + + +class VolumeGroupsModel(object): + def __init__(self, **kargs): + pass + + def get_list(self): + return [vg['vgname'] for vg in disks.vgs()] + + +class VolumeGroupModel(object): + def __init__(self, **kargs): + pass + + def lookup(self, name): + def _format(vg): + return {'name': vg['vgname'], + 'size': vg['size'], + 'free': vg['free'], + 'pvs': [pv['pvname'] for pv in disks.pvs(vg['vgname'])], + 'lvs': [lv['lvname'] for lv in disks.lvs(vg['vgname'])]} + + vgs = [_format(vg) for vg in disks.vgs() if vg['vgname'] == name] + if not vgs: + raise InvalidParameter("KCHLVMS0001E", {'name': name}) + + return vgs[0] diff --git a/src/wok/plugins/kimchi/model/libvirtstoragepool.py b/src/wok/plugins/kimchi/model/libvirtstoragepool.py index 5da0b3b..0fa8ce0 100644 --- a/src/wok/plugins/kimchi/model/libvirtstoragepool.py +++ b/src/wok/plugins/kimchi/model/libvirtstoragepool.py @@ -144,11 +144,12 @@ class LogicalPoolDef(StoragePoolDef): pool = E.pool(type='logical') pool.append(E.name(self.poolArgs['name'])) - source = E.source() - for device_path in self.poolArgs['source']['devices']: - source.append(E.device(path=device_path)) + if not self.poolArgs['source'].get('from_vg', False): + source = E.source() + for device_path in self.poolArgs['source']['devices']: + source.append(E.device(path=device_path)) + pool.append(source) - pool.append(source) pool.append(E.target(E.path(self.path))) return ET.tostring(pool, encoding='unicode', pretty_print=True) diff --git a/src/wok/plugins/kimchi/model/storagepools.py b/src/wok/plugins/kimchi/model/storagepools.py index cc0bc7c..ec866c5 100644 --- a/src/wok/plugins/kimchi/model/storagepools.py +++ b/src/wok/plugins/kimchi/model/storagepools.py @@ -142,9 +142,27 @@ class StoragePoolsModel(object): raise OperationFailed("KCHPOOL0006E", {'err': e.get_error_message()}) + def _check_lvm(self, name, from_vg): + vgdisplay_cmd = ['vgdisplay', name.encode('utf-8')] + output, error, returncode = run_command(vgdisplay_cmd) + # From vgdisplay error codes: + # 1 error reading VGDA + # 2 volume group doesn't exist + # 3 not all physical volumes of volume group online + # 4 volume group not found + # 5 no volume groups found at all + # 6 error reading VGDA from lvmtab + if from_vg and returncode in [2, 4, 5]: + raise InvalidOperation("KCHPOOL0038E", {'name': name}) + + if not from_vg and returncode not in [2, 4, 5]: + raise InvalidOperation("KCHPOOL0036E", {'name': name}) + def create(self, params): task_id = None conn = self.conn.get() + from_vg = params.get('source', {}).get('from_vg', False) + try: name = params['name'] if name == ISO_POOL_NAME: @@ -154,17 +172,7 @@ class StoragePoolsModel(object): # used before but a volume group will already exist with this name # So check the volume group does not exist to create the pool if params['type'] == 'logical': - vgdisplay_cmd = ['vgdisplay', name.encode('utf-8')] - output, error, returncode = run_command(vgdisplay_cmd) - # From vgdisplay error codes: - # 1 error reading VGDA - # 2 volume group doesn't exist - # 3 not all physical volumes of volume group online - # 4 volume group not found - # 5 no volume groups found at all - # 6 error reading VGDA from lvmtab - if returncode not in [2, 4, 5]: - raise InvalidOperation("KCHPOOL0036E", {'name': name}) + self._check_lvm(name, from_vg) if params['type'] == 'kimchi-iso': task_id = self._do_deep_scan(params) diff --git a/src/wok/plugins/kimchi/tests/test_mock_storagepool.py b/src/wok/plugins/kimchi/tests/test_mock_storagepool.py index 29488b8..f017f37 100644 --- a/src/wok/plugins/kimchi/tests/test_mock_storagepool.py +++ b/src/wok/plugins/kimchi/tests/test_mock_storagepool.py @@ -65,6 +65,10 @@ class MockStoragepoolTests(unittest.TestCase): ) def test_storagepool(self): + # MockModel always returns 2 VGs (hostVG, kimchiVG) + vgs = json.loads(self.request('/plugins/kimchi/host/vgs').read()) + vg_names = [vg['name'] for vg in vgs] + # MockModel always returns 2 partitions (vdx, vdz) partitions = json.loads( self.request('/plugins/kimchi/host/partitions').read() @@ -89,7 +93,9 @@ class MockStoragepoolTests(unittest.TestCase): 'source': {'host': '127.0.0.1', 'target': 'iqn.2015-01.localhost.kimchiUnitTest'}}, {'type': 'logical', 'name': u'kīмсhīUnitTestLogicalPool', - 'source': {'devices': [devs[0]]}}] + 'source': {'devices': [devs[0]]}}, + {'type': 'logical', 'name': vg_names[0], + 'source': {'from_vg': True}}] def _do_test(params): name = params['name'] -- 2.5.0