Re: [Kimchi-devel] [PATCH] [Kimchi] Move Kimchi specific functions from gingerbase.disks to Kimchi

The idea behind this patch is to eliminate the Ginger Base dependency. Some functions in gingerbase.disks are for only Kimchi matters. Others are shared with Ginger and can not be rewritten using PyParted, each means they will be place on Kimchi and Ginger Base. As this code is mature enough, further changes will have few impact and the gain to incorporate Ginger Base into Ginger and eliminate a Kimchi dependency is bigger. Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com&gt; --- disks.py | 325 ++++++++++++++++++++++++++++++++++++++++++++++++++++ i18n.py | 5 + model/host.py | 4 +- tests/test_disks.py | 53 +++++++++ 4 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 disks.py create mode 100644 tests/test_disks.py diff --git a/disks.py b/disks.py new file mode 100644 index 0000000..e86ca2f --- /dev/null +++ b/disks.py @@ -0,0 +1,325 @@ +# +# Project Kimchi +# +# Copyright IBM Corp, 2015-2017 +# +# Code derived from Ginger Base project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os.path +import re +from parted import Device as PDevice +from parted import Disk as PDisk + +from wok.exception import NotFoundError, OperationFailed +from wok.stringutils import encode_value +from wok.utils import run_command, wok_log + + +def _get_dev_node_path(maj_min): + """ Returns device node path given the device number 'major:min' """ + + dm_name = "/sys/dev/block/%s/dm/name" % maj_min + if os.path.exists(dm_name): + with open(dm_name) as dm_f: + content = dm_f.read().rstrip('\n') + return "/dev/mapper/" + content + + uevent = "/sys/dev/block/%s/uevent" % maj_min + with open(uevent) as ueventf: + content = ueventf.read() + + data = dict(re.findall(r'(\S+)=(".*?"|\S+)', content.replace("\n", " "))) + + return "/dev/%s" % data["DEVNAME"] + + +def _get_lsblk_devs(keys, devs=None): + if devs is None: + devs = [] + out, err, returncode = run_command( + ["lsblk", "-Pbo"] + [','.join(keys)] + devs + ) + if returncode != 0: + if 'not a block device' in err: + raise NotFoundError("KCHDISK00002E") + else: + raise OperationFailed("KCHDISK00001E", {'err': err}) + + return _parse_lsblk_output(out, keys) + + +def _get_dev_major_min(name): + maj_min = None + + keys = ["NAME", "MAJ:MIN"] + try: + dev_list = _get_lsblk_devs(keys) + except: + raise + + for dev in dev_list: + if dev['name'].split()[0] == name: + maj_min = dev['maj:min'] + break + else: + raise NotFoundError("KCHDISK00003E", {'device': name}) + + return maj_min + + +def _is_dev_leaf(devNodePath, name=None, devs=None, devtype=None): + try: + if devs and devtype != 'mpath': + for dev in devs: + if encode_value(name) == dev['pkname']: + return False + return True + # By default, lsblk prints a device information followed by children + # device information + childrenCount = len( + _get_lsblk_devs(["NAME"], [devNodePath])) - 1 + except OperationFailed as e: + # lsblk is known to fail on multipath devices + # Assume these devices contain children + wok_log.error( + "Error getting device info for %s: %s", devNodePath, e) + return False + + return childrenCount == 0 + + +def _is_dev_extended_partition(devType, devNodePath): + if devType != 'part': + return False + + if devNodePath.startswith('/dev/mapper'): + try: + dev_maj_min = _get_dev_major_min(devNodePath.split("/")[-1]) + parent_sys_path = '/sys/dev/block/' + dev_maj_min + '/slaves' + parent_dm_name = os.listdir(parent_sys_path)[0] + parent_maj_min = open( + parent_sys_path + + '/' + + parent_dm_name + + '/dev').readline().rstrip() + diskPath = _get_dev_node_path(parent_maj_min) + except Exception as e: + wok_log.error( + "Error dealing with dev mapper device: " + devNodePath) + raise OperationFailed("KCHDISK00001E", {'err': e.message}) + else: + diskPath = devNodePath.rstrip('0123456789') + + device = PDevice(diskPath) + try: + extended_part = PDisk(device).getExtendedPartition() + except NotImplementedError as e: + wok_log.warning( + "Error getting extended partition info for dev %s type %s: %s", + devNodePath, devType, e.message) + # Treate disk with unsupported partiton table as if it does not + # contain extended partitions. + return False + if extended_part and extended_part.path == devNodePath: + return True + return False + + +def _parse_lsblk_output(output, keys): + # output is on format key="value", + # where key can be NAME, TYPE, FSTYPE, SIZE, MOUNTPOINT, etc + lines = output.rstrip("\n").split("\n") + r = [] + for line in lines: + d = {} + for key in keys: + expression = r"%s=\".*?\"" % key + match = re.search(expression, line) + field = match.group() + k, v = field.split('=', 1) + d[k.lower()] = v[1:-1] + r.append(d) + return r + + +def _is_available(name, devtype, fstype, mountpoint, majmin, devs=None): + devNodePath = _get_dev_node_path(majmin) + # Only list unmounted and unformated and leaf and (partition or disk) + # leaf means a partition, a disk has no partition, or a disk not held + # by any multipath device. Physical volume belongs to no volume group + # is also listed. Extended partitions should not be listed. + if fstype == 'LVM2_member': + has_VG = True + else: + has_VG = False + if (devtype in ['part', 'disk', 'mpath'] and + fstype in ['', 'LVM2_member'] and + mountpoint == "" and + not has_VG and + _is_dev_leaf(devNodePath, name, devs, devtype) and + not _is_dev_extended_partition(devtype, devNodePath)): + return True + return False + + +def get_partitions_names(check=False): + names = set() + keys = ["NAME", "TYPE", "FSTYPE", "MOUNTPOINT", "MAJ:MIN"] + # output is on format key="value", + # where key can be NAME, TYPE, FSTYPE, MOUNTPOINT + for dev in _get_lsblk_devs(keys): + # split()[0] to avoid the second part of the name, after the + # whiteline + name = dev['name'].split()[0] + if check and not _is_available(name, dev['type'], dev['fstype'], + dev['mountpoint'], dev['maj:min']): + continue + names.add(name) + + return list(names) + + +def get_partition_details(name): + majmin = _get_dev_major_min(name) + dev_path = _get_dev_node_path(majmin) + + keys = ["TYPE", "FSTYPE", "SIZE", "MOUNTPOINT", "MAJ:MIN", "PKNAME"] + try: + dev = _get_lsblk_devs(keys, [dev_path])[0] + except: + wok_log.error("Error getting partition info for %s", name) + return {} + + dev['available'] = _is_available(name, dev['type'], dev['fstype'], + dev['mountpoint'], majmin) + if dev['mountpoint']: + # Sometimes the mountpoint comes with [SWAP] or other + # info which is not an actual mount point. Filtering it + regexp = re.compile(r"\[.*\]") + if regexp.search(dev['mountpoint']) is not None: + dev['mountpoint'] = '' + dev['path'] = dev_path + dev['name'] = name + return dev + + +def vgs(): + """ + lists all volume groups in the system. All size units are in bytes. + + [{'vgname': 'vgtest', 'size': 999653638144L, 'free': 0}] + """ + cmd = ['vgs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'vg_name,vg_size,vg_free'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHDISK00004E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of VGs + vgs = map(lambda v: v.strip(), out.strip('\n').split('\n')) + + # create a dict based on data retrieved from vgs + return map(lambda l: {'vgname': l[0], + 'size': long(l[1]), + 'free': long(l[2])}, + [fields.split() for fields in vgs]) + + +def lvs(vgname=None): + """ + lists all logical volumes found in the system. It can be filtered by + the volume group. All size units are in bytes. + + [{'lvname': 'lva', 'path': '/dev/vgtest/lva', 'size': 12345L}, + {'lvname': 'lvb', 'path': '/dev/vgtest/lvb', 'size': 12345L}] + """ + cmd = ['lvs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'lv_name,lv_path,lv_size,vg_name'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHDISK00004E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of LVs filtered by vgname, if + # provided + lvs = filter(lambda f: vgname is None or vgname in f, + map(lambda v: v.strip(), out.strip('\n').split('\n'))) + + # create a dict based on data retrieved from lvs + return map(lambda l: {'lvname': l[0], + 'path': l[1], + 'size': long(l[2])}, + [fields.split() for fields in lvs]) + + +def pvs(vgname=None): + """ + lists all physical volumes in the system. It can be filtered by the + volume group. All size units are in bytes. + + [{'pvname': '/dev/sda3', + 'size': 469502001152L, + 'uuid': 'kkon5B-vnFI-eKHn-I5cG-Hj0C-uGx0-xqZrXI'}, + {'pvname': '/dev/sda2', + 'size': 21470642176L, + 'uuid': 'CyBzhK-cQFl-gWqr-fyWC-A50Y-LMxu-iHiJq4'}] + """ + cmd = ['pvs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'pv_name,pv_size,pv_uuid,vg_name'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHDISK00004E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of PVs filtered by vgname, if + # provided + pvs = filter(lambda f: vgname is None or vgname in f, + map(lambda v: v.strip(), out.strip('\n').split('\n'))) + + # create a dict based on data retrieved from pvs + return map(lambda l: {'pvname': l[0], + 'size': long(l[1]), + 'uuid': l[2]}, + [fields.split() for fields in pvs]) diff --git a/i18n.py b/i18n.py index 4460ce5..7dc7b05 100644 --- a/i18n.py +++ b/i18n.py @@ -29,6 +29,11 @@ messages = { "KCHPART0001E": _("Partition %(name)s does not exist in the host"), + "KCHDISK00001E": _("Error while accessing dev mapper device, %(err)s"), + "KCHDISK00002E": _("Block device not found."), + "KCHDISK00003E": _("Block device %(device)s not found."), + "KCHDISK00004E": _("Unable to retrieve LVM information. Details: %(err)s"), + "KCHDEVS0001E": _('Unknown "_cap" specified'), "KCHDEVS0002E": _('"_passthrough" should be "true" or "false"'), "KCHDEVS0003E": _('"_passthrough_affected_by" should be a device name string'), diff --git a/model/host.py b/model/host.py index abf1191..90834e3 100644 --- a/model/host.py +++ b/model/host.py @@ -1,7 +1,7 @@ # # Project Kimchi # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -26,7 +26,7 @@ from wok.exception import InvalidParameter from wok.exception import NotFoundError from wok.xmlutils.utils import xpath_get_text -from wok.plugins.gingerbase import disks +from wok.plugins.kimchi import disks from wok.plugins.kimchi.model import hostdev from wok.plugins.kimchi.model.config import CapabilitiesModel from wok.plugins.kimchi.model.vms import VMModel, VMsModel diff --git a/tests/test_disks.py b/tests/test_disks.py new file mode 100644 index 0000000..6782f55 --- /dev/null +++ b/tests/test_disks.py @@ -0,0 +1,53 @@ +# +# Project Kimchi +# +# Copyright IBM Corp, 2017 +# +# Code derived from Ginger Base +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import mock +import unittest + +from wok.exception import NotFoundError, OperationFailed +from wok.plugins.kimchi.disks import _get_lsblk_devs + + +class DiskTests(unittest.TestCase): + + @mock.patch('wok.plugins.kimchi.disks.run_command') + def test_lsblk_returns_404_when_device_not_found(self, mock_run_command): + mock_run_command.return_value = ["", "not a block device", 32] + fake_dev = "/not/a/true/block/dev" + keys = ["MOUNTPOINT"] + + with self.assertRaises(NotFoundError): + _get_lsblk_devs(keys, [fake_dev]) + cmd = ['lsblk', '-Pbo', 'MOUNTPOINT', fake_dev] + mock_run_command.assert_called_once_with(cmd) + + @mock.patch('wok.plugins.kimchi.disks.run_command') + def test_lsblk_returns_500_when_unknown_error_occurs( + self, mock_run_command): + + mock_run_command.return_value = ["", "", 1] + valid_dev = "/valid/block/dev" + keys = ["MOUNTPOINT"] + + with self.assertRaises(OperationFailed): + _get_lsblk_devs(keys, [valid_dev]) + cmd = ['lsblk', '-Pbo', 'MOUNTPOINT', valid_dev] + mock_run_command.assert_called_once_with(cmd)