The URI /host/devices only presents scsi_host (particularly fc_host) device information. To implement host PCI pass through, we should list all types of host devices. This patch adds support for parsing various host devices information, and listing them on /host/devices. So the user is free to choose any listed PCI device to pass through to guest. Since the patch changes the device information dictionary format, the existing code consuming the device information is also changed accordingly. To get all types of host device, access the following URL. curl -k -u root -H "Content-Type: application/json" \ -H "Accept: application/json" \ https://127.0.0.1:8001/host/devices To get only fc_host devices, change the URL to "https://127.0.0.1:8001/host/devices?_cap=fc_host" To get only pci device, change the URL to "https://127.0.0.1:8001/host/devices?_cap=pci" v1: Parse the node device XML using xpath. v2: Write a "dictize" function and parse the node device XML using dictize. v3: Fix a naming mistake. v4: It is observed that sometimes the parent devices is not listed by libvirt but the child device is listed. In previous version we catch this exception and ignore it. The root cause is unknown, and we failed to re-produce the problem. In v4 we do not catch it. It seems to be related to USB removable disk, and the problem is gone after we upgraded Linux kernel. Signed-off-by: Zhou Zheng Sheng <zhshzhou(a)linux.vnet.ibm.com&gt; --- docs/API.md | 11 +- src/kimchi/hostdev.py | 209 +++++++++++++++++++++++++++++++++ src/kimchi/mockmodel.py | 7 +- src/kimchi/model/host.py | 15 +-- src/kimchi/model/libvirtstoragepool.py | 18 +-- src/kimchi/xmlutils.py | 26 +++- tests/test_rest.py | 6 +- tests/test_storagepool.py | 7 +- 8 files changed, 262 insertions(+), 37 deletions(-) create mode 100644 src/kimchi/hostdev.py diff --git a/docs/API.md b/docs/API.md index 9217a37..484a4c8 100644 --- a/docs/API.md +++ b/docs/API.md @@ -869,12 +869,11 @@ stats history * **GET**: Retrieve information of a single pci device. Currently only scsi_host devices are supported: * name: The name of the device. - * adapter_type: The capability type of the scsi_host device (fc_host). - Empty if pci device is not scsi_host. - * wwnn: The HBA Word Wide Node Name. - Empty if pci device is not scsi_host. - * wwpn: The HBA Word Wide Port Name - Empty if pci device is not scsi_host. + * path: Path of device in sysfs. + * adapter: Host adapter information. Empty if pci device is not scsi_host. + * type: The capability type of the scsi_host device (fc_host, vport_ops). + * wwnn: The HBA Word Wide Node Name. Empty if pci device is not fc_host. + * wwpn: The HBA Word Wide Port Name. Empty if pci device is not fc_host. ### Collection: Host Packages Update diff --git a/src/kimchi/hostdev.py b/src/kimchi/hostdev.py new file mode 100644 index 0000000..d4c142d --- /dev/null +++ b/src/kimchi/hostdev.py @@ -0,0 +1,209 @@ +# +# Kimchi +# +# Copyright IBM Corp, 2014 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from kimchi.model.libvirtconnection import LibvirtConnection +from kimchi.utils import kimchi_log +from kimchi.xmlutils import dictize + + +def _get_all_host_dev_infos(): + libvirt_conn = LibvirtConnection('qemu:///system').get() + node_devs = libvirt_conn.listAllDevices() + return [get_dev_info(node_dev) for node_dev in node_devs] + + +def _get_dev_info_tree(dev_infos): + devs = dict([(dev_info['name'], dev_info) for dev_info in dev_infos]) + root = None + for dev_info in dev_infos: + if dev_info['parent'] is None: + root = dev_info + continue + parent = devs[dev_info['parent']] + + try: + children = parent['children'] + except KeyError: + parent['children'] = [dev_info] + else: + children.append(dev_info) + return root + + +def get_dev_info(node_dev): + ''' Parse the node device XML string into dict according to + http://libvirt.org/formatnode.html. ''' + + def shift_subdict(d, toshift): + subdict = d.pop(toshift) + d.update(subdict) + return d + + xmlstr = node_dev.XMLDesc() + info = dictize(xmlstr)['device'] + dev_type = info['capability'].pop('type') + info['device_type'] = dev_type + shift_subdict(info, 'capability') + info['parent'] = node_dev.parent() + + get_dev_type_info = { + 'net': _get_net_dev_info, + 'pci': _get_pci_dev_info, + 'scsi': _get_scsi_dev_info, + 'scsi_generic': _get_scsi_generic_dev_info, + 'scsi_host': _get_scsi_host_dev_info, + 'scsi_target': _get_scsi_target_dev_info, + 'storage': _get_storage_dev_info, + 'system': _get_system_dev_info, + 'usb': _get_usb_dev_info, + 'usb_device': _get_usb_device_dev_info, + } + try: + get_detail_info = get_dev_type_info[dev_type] + except KeyError: + kimchi_log.error("Unknown device type: %s", dev_type) + return info + + return get_detail_info(info) + + +def _get_net_dev_info(info): + cap = info.pop('capability') + links = {"80203": "IEEE 802.3", "80211": "IEEE 802.11"} + link_raw = cap['type'] + info['link_type'] = links.get(link_raw, link_raw) + + return info + + +def _get_pci_dev_info(info): + for k in ('vendor', 'product'): + info[k]['description'] = info[k].pop('pyval') + try: + info['iommuGroup'] = info['iommuGroup']['number'] + except KeyError: + # No IOMMU group support. + pass + return info + + +def _get_scsi_dev_info(info): + return info + + +def _get_scsi_generic_dev_info(info): + # scsi_generic is not documented in libvirt official website. Try to + # parse scsi_generic according to the following libvirt path series. + # https://www.redhat.com/archives/libvir-list/2013-June/msg00014.html + return info + + +def _get_scsi_host_dev_info(info): + try: + cap_info = info.pop('capability') + except KeyError: + # kimchi.model.libvirtstoragepool.ScsiPoolDef assumes + # info['adapter']['type'] always exists. + info['adapter'] = {'type': ''} + return info + info['adapter'] = cap_info + return info + + +def _get_scsi_target_dev_info(info): + # scsi_target is not documented in libvirt official website. Try to + # parse scsi_target according to the libvirt commit db19834a0a. + return info + + +def _get_storage_dev_info(info): + try: + cap_info = info.pop('capability') + except KeyError: + return info + + if cap_info['type'] == 'removable': + cap_info['available'] = bool(cap_info.pop('media_available')) + if cap_info['available']: + cap_info.update({'size': cap_info.pop('media_size'), + 'label': cap_info.pop('media_label')}) + info['media'] = cap_info + return info + + +def _get_system_dev_info(info): + return info + + +def _get_usb_dev_info(info): + return info + + +def _get_usb_device_dev_info(info): + for k in ('vendor', 'product'): + try: + info[k]['description'] = info[k].pop('pyval') + except KeyError: + # Some USB devices don't provide vendor/product description. + pass + return info + + +# For test and debug +def _print_host_dev_tree(): + dev_infos = _get_all_host_dev_infos() + root = _get_dev_info_tree(dev_infos) + if root is None: + print "No device found" + return + print '-----------------' + print '\n'.join(_format_dev_node(root)) + + +def _format_dev_node(node): + from pprint import pformat + + try: + children = node['children'] + del node['children'] + except KeyError: + children = [] + + lines = [] + lines.extend([' ~' + line for line in pformat(node).split('\n')]) + + count = len(children) + for i, child in enumerate(children): + if count == 1: + lines.append(' \-----------------') + else: + lines.append(' +-----------------') + clines = _format_dev_node(child) + if i == count - 1: + p = ' ' + else: + p = ' |' + lines.extend([p + cline for cline in clines]) + lines.append('') + + return lines + + +if __name__ == '__main__': + _print_host_dev_tree() diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py index 05720f4..7420e7e 100644 --- a/src/kimchi/mockmodel.py +++ b/src/kimchi/mockmodel.py @@ -493,9 +493,10 @@ class MockModel(object): def device_lookup(self, nodedev_name): return { 'name': nodedev_name, - 'adapter_type': 'fc_host', - 'wwnn': uuid.uuid4().hex[:16], - 'wwpn': uuid.uuid4().hex[:16]} + 'adapter': { + 'type': 'fc_host', + 'wwnn': uuid.uuid4().hex[:16], + 'wwpn': uuid.uuid4().hex[:16]}} def isopool_lookup(self, name): return {'state': 'active', diff --git a/src/kimchi/model/host.py b/src/kimchi/model/host.py index e9ac487..f4bd613 100644 --- a/src/kimchi/model/host.py +++ b/src/kimchi/model/host.py @@ -28,6 +28,7 @@ import psutil from cherrypy.process.plugins import BackgroundTask from kimchi import disks +from kimchi import hostdev from kimchi import netinfo from kimchi import xmlutils from kimchi.basemodel import Singleton @@ -285,20 +286,10 @@ class DeviceModel(object): def lookup(self, nodedev_name): conn = self.conn.get() try: - dev_xml = conn.nodeDeviceLookupByName(nodedev_name).XMLDesc(0) + dev = conn.nodeDeviceLookupByName(nodedev_name) except: raise NotFoundError('KCHHOST0003E', {'name': nodedev_name}) - cap_type = xmlutils.xpath_get_text( - dev_xml, '/device/capability/capability/@type') - wwnn = xmlutils.xpath_get_text( - dev_xml, '/device/capability/capability/wwnn') - wwpn = xmlutils.xpath_get_text( - dev_xml, '/device/capability/capability/wwpn') - return { - 'name': nodedev_name, - 'adapter_type': cap_type[0] if len(cap_type) >= 1 else '', - 'wwnn': wwnn[0] if len(wwnn) == 1 else '', - 'wwpn': wwpn[0] if len(wwpn) == 1 else ''} + return hostdev.get_dev_info(dev) class PackagesUpdateModel(object): diff --git a/src/kimchi/model/libvirtstoragepool.py b/src/kimchi/model/libvirtstoragepool.py index 47b239b..b15bf1a 100644 --- a/src/kimchi/model/libvirtstoragepool.py +++ b/src/kimchi/model/libvirtstoragepool.py @@ -180,34 +180,34 @@ class ScsiPoolDef(StoragePoolDef): self.poolArgs['source']['name'] = tmp_name.replace('scsi_', '') # fc_host adapters type are only available in libvirt >= 1.0.5 if not self.poolArgs['fc_host_support']: - self.poolArgs['source']['adapter_type'] = 'scsi_host' + self.poolArgs['source']['adapter']['type'] = 'scsi_host' msg = "Libvirt version <= 1.0.5. Setting SCSI host name as '%s'; "\ "setting SCSI adapter type as 'scsi_host'; "\ "ignoring wwnn and wwpn." % tmp_name kimchi_log.info(msg) # Path for Fibre Channel scsi hosts self.poolArgs['path'] = '/dev/disk/by-path' - if not self.poolArgs['source']['adapter_type']: - self.poolArgs['source']['adapter_type'] = 'scsi_host' + if not self.poolArgs['source']['adapter']['type']: + self.poolArgs['source']['adapter']['type'] = 'scsi_host' @property def xml(self): # Required parameters # name: - # source[adapter_type]: + # source[adapter][type]: # source[name]: - # source[wwnn]: - # source[wwpn]: + # source[adapter][wwnn]: + # source[adapter][wwpn]: # path: xml = """ <pool type='scsi'> <name>{name}</name> <source> - <adapter type='{source[adapter_type]}'\ + <adapter type='{source[adapter][type]}'\ name='{source[name]}'\ - wwnn='{source[wwnn]}'\ - wwpn='{source[wwpn]}'/> + wwnn='{source[adapter][wwnn]}'\ + wwpn='{source[adapter][wwpn]}'/> </source> <target> <path>{path}</path> diff --git a/src/kimchi/xmlutils.py b/src/kimchi/xmlutils.py index 76f0696..56517f2 100644 --- a/src/kimchi/xmlutils.py +++ b/src/kimchi/xmlutils.py @@ -18,6 +18,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import libxml2 +from lxml import objectify from xml.etree import ElementTree @@ -26,7 +27,7 @@ from xml.etree import ElementTree def xpath_get_text(xml, expr): doc = libxml2.parseDoc(xml) res = doc.xpathEval(expr) - ret = [None if x.children == None else x.children.content for x in res] + ret = [None if x.children is None else x.children.content for x in res] doc.freeDoc() return ret @@ -37,3 +38,26 @@ def xml_item_update(xml, xpath, value): item = root.find(xpath) item.text = value return ElementTree.tostring(root, encoding="utf-8") + + +def dictize(xmlstr): + root = objectify.fromstring(xmlstr) + return {root.tag: _dictize(root)} + + +def _dictize(e): + d = {} + if e.text is not None: + if not e.attrib and e.countchildren() == 0: + return e.pyval + d['pyval'] = e.pyval + d.update(e.attrib) + for child in e.iterchildren(): + if child.tag in d: + continue + if len(child) > 1: + d[child.tag] = [ + _dictize(same_tag_child) for same_tag_child in child] + else: + d[child.tag] = _dictize(child) + return d diff --git a/tests/test_rest.py b/tests/test_rest.py index 7ed94cb..cb1ae9a 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -158,9 +158,9 @@ class RestTests(unittest.TestCase): nodedev = json.loads(self.request('/host/devices/scsi_host4').read()) # Mockmodel generates random wwpn and wwnn self.assertEquals('scsi_host4', nodedev['name']) - self.assertEquals('fc_host', nodedev['adapter_type']) - self.assertEquals(16, len(nodedev['wwpn'])) - self.assertEquals(16, len(nodedev['wwnn'])) + self.assertEquals('fc_host', nodedev['adapter']['type']) + self.assertEquals(16, len(nodedev['adapter']['wwpn'])) + self.assertEquals(16, len(nodedev['adapter']['wwnn'])) def test_get_vms(self): vms = json.loads(self.request('/vms').read()) diff --git a/tests/test_storagepool.py b/tests/test_storagepool.py index 22b4943..3e3ad83 100644 --- a/tests/test_storagepool.py +++ b/tests/test_storagepool.py @@ -145,9 +145,10 @@ class storagepoolTests(unittest.TestCase): 'path': '/dev/disk/by-path', 'source': { 'name': 'scsi_host3', - 'adapter_type': 'fc_host', - 'wwpn': '0123456789abcdef', - 'wwnn': 'abcdef0123456789'}}, + 'adapter': { + 'type': 'fc_host', + 'wwpn': '0123456789abcdef', + 'wwnn': 'abcdef0123456789'}}}, 'xml': """ <pool type='scsi'> -- 1.9.3

This patch enbales Kimchi's VM to use host devices directly, and it greatly improves the related device performance. The user can assign PCI, USB and SCSI LUN directly to VM, as long as the host supports one of Intel VT-d, AMD IOMMU or POWER sPAPR technology and runs a recent release of Linux kernel. This patch adds a sub-collection "hostdevs" to the URI vms/vm-name/. The front-end can GET vms/vm-name/hostdevs and vms/vm-name/hostdevs/dev-name or POST (assign) vms/vm-name/hostdevs and DELETE (dismiss) vms/vm-name/hostdevs/dev-name The eligible devices to assign are the devices listed by the URI host/devices?_passthrough=1 When assigning a host PCI device to VM, all the eligible PCI devices in the same IOMMU group are also automatically assigned, and vice versa when dismissing a host PIC device from the VM. Some examples: Assign a USB device: curl -k -u root -H "Content-Type: application/json" \ -H "Accept: application/json" \ -X POST -d '{"name": "usb_1_1_6"}' \ 'https://127.0.0.1:8001/vms/rhel65/hostdevs' Assign a PCI device: -d '{"name": "pci_0000_0d_00_0"}' Assign a SCSI LUN: -d '{"name": "scsi_1_0_0_0"}' List assigned devices: curl -k -u root -H "Content-Type: application/json" \ -H "Accept: application/json" \ 'https://127.0.0.1:8001/vms/rhel65/hostdevs' The above command should print following. [ { "type":"scsi", "name":"scsi_1_0_0_0" }, { "type":"usb", "name":"usb_1_1_6" }, { "type":"pci", "name":"pci_0000_0d_00_0" }, { "type":"pci", "name":"pci_0000_03_00_0" } ] Notice that the device pci_0000_03_00_0 is also assigned automatically. The assigned devices are hot-plugged to VM and also written to the domain XML. When it's possible, it enables VFIO for PCI device assignment. v1: Handle the devices in the VM template. v2: Handle the devices in the VM sub-resource "hostdevs". v3: No change. v4: Not all domain XMLs contain hostdev node. Deal with the case. v5: Change _passthrough='1' to _passthrough='true'. When attaching and detaching a device, do not use VIR_DOMAIN_AFFECT_CURRENT flag, instead, use kimchi.model.utils.get_vm_config_flag() to correctly set the device flag. Signed-off-by: Zhou Zheng Sheng <zhshzhou(a)linux.vnet.ibm.com&gt; --- src/kimchi/control/vm/hostdevs.py | 44 ++++++ src/kimchi/featuretests.py | 10 +- src/kimchi/i18n.py | 7 + src/kimchi/model/config.py | 2 + src/kimchi/model/vmhostdevs.py | 303 ++++++++++++++++++++++++++++++++++++++ src/kimchi/rollbackcontext.py | 3 + 6 files changed, 368 insertions(+), 1 deletion(-) create mode 100644 src/kimchi/control/vm/hostdevs.py create mode 100644 src/kimchi/model/vmhostdevs.py diff --git a/src/kimchi/control/vm/hostdevs.py b/src/kimchi/control/vm/hostdevs.py new file mode 100644 index 0000000..81fe8ec --- /dev/null +++ b/src/kimchi/control/vm/hostdevs.py @@ -0,0 +1,44 @@ +# +# Project Kimchi +# +# Copyright IBM, Corp. 2014 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from kimchi.control.base import Collection, Resource +from kimchi.control.utils import UrlSubNode + + +@UrlSubNode("hostdevs") +class VMHostDevs(Collection): + def __init__(self, model, vmid): + super(VMHostDevs, self).__init__(model) + self.resource = VMHostDev + self.vmid = vmid + self.resource_args = [self.vmid, ] + self.model_args = [self.vmid, ] + + +class VMHostDev(Resource): + def __init__(self, model, vmid, ident): + super(VMHostDev, self).__init__(model, ident) + self.vmid = vmid + self.ident = ident + self.info = {} + self.model_args = [self.vmid, self.ident] + + @property + def data(self): + return self.info diff --git a/src/kimchi/featuretests.py b/src/kimchi/featuretests.py index 5192361..74222bf 100644 --- a/src/kimchi/featuretests.py +++ b/src/kimchi/featuretests.py @@ -29,7 +29,7 @@ from lxml.builder import E from kimchi.rollbackcontext import RollbackContext -from kimchi.utils import kimchi_log +from kimchi.utils import kimchi_log, run_command ISO_STREAM_XML = """ @@ -206,3 +206,11 @@ class FeatureTests(object): return True except libvirt.libvirtError: return False + + @staticmethod + def kernel_support_vfio(): + out, err, rc = run_command(['modprobe', 'vfio-pci']) + if rc != 0: + kimchi_log.warning("Unable to load Kernal module vfio-pci.") + return False + return True diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index 25b703f..3c31ab6 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -89,6 +89,13 @@ messages = { "KCHVM0029E": _("Unable to shutdown virtual machine %(name)s. Details: %(err)s"), "KCHVM0030E": _("Unable to get access metadata of virtual machine %(name)s. Details: %(err)s"), + "KCHVMHDEV0001E": _("VM %(vmid)s does not contain directly assigned host device %(dev_name)s."), + "KCHVMHDEV0002E": _("The host device %(dev_name)s is not allowed to directly assign to VM."), + "KCHVMHDEV0003E": _("No IOMMU groups found. Host PCI pass through needs IOMMU group to function correctly. " + "Please enable Intel VT-d or AMD IOMMU in your BIOS, then verify the Kernel is compiled with IOMMU support. " + "For Intel CPU, add intel_iommu=on to your Kernel parameter in /boot/grub2/grub.conf. " + "For AMD CPU, add iommu=pt iommu=1."), + "KCHVMIF0001E": _("Interface %(iface)s does not exist in virtual machine %(name)s"), "KCHVMIF0002E": _("Network %(network)s specified for virtual machine %(name)s does not exist"), "KCHVMIF0003E": _("Do not support guest interface hot plug attachment"), diff --git a/src/kimchi/model/config.py b/src/kimchi/model/config.py index 0ef0855..95c8e7e 100644 --- a/src/kimchi/model/config.py +++ b/src/kimchi/model/config.py @@ -54,6 +54,7 @@ class CapabilitiesModel(object): self.libvirt_stream_protocols = [] self.fc_host_support = False self.metadata_support = False + self.kernel_vfio = False # Subscribe function to set host capabilities to be run when cherrypy # server is up @@ -67,6 +68,7 @@ class CapabilitiesModel(object): self.nfs_target_probe = FeatureTests.libvirt_support_nfs_probe() self.fc_host_support = FeatureTests.libvirt_support_fc_host() self.metadata_support = FeatureTests.has_metadata_support() + self.kernel_vfio = FeatureTests.kernel_support_vfio() self.libvirt_stream_protocols = [] for p in ['http', 'https', 'ftp', 'ftps', 'tftp']: diff --git a/src/kimchi/model/vmhostdevs.py b/src/kimchi/model/vmhostdevs.py new file mode 100644 index 0000000..9e59513 --- /dev/null +++ b/src/kimchi/model/vmhostdevs.py @@ -0,0 +1,303 @@ +# +# Project Kimchi +# +# Copyright IBM, Corp. 2014 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import glob + +import libvirt +from lxml import etree, objectify + +from kimchi.exception import InvalidOperation, InvalidParameter, NotFoundError +from kimchi.model.config import CapabilitiesModel +from kimchi.model.host import DeviceModel, DevicesModel +from kimchi.model.host import PassthroughAffectedDevicesModel +from kimchi.model.utils import get_vm_config_flag +from kimchi.model.vms import DOM_STATE_MAP, VMModel +from kimchi.rollbackcontext import RollbackContext +from kimchi.utils import kimchi_log, run_command + + +class VMHostDevsModel(object): + def __init__(self, **kargs): + self.conn = kargs['conn'] + + def get_list(self, vmid): + dom = VMModel.get_vm(vmid, self.conn) + xmlstr = dom.XMLDesc(0) + root = objectify.fromstring(xmlstr) + try: + hostdev = root.devices.hostdev + except AttributeError: + return [] + + return [self._deduce_dev_name(e) for e in hostdev] + + @staticmethod + def _toint(num_str): + if num_str.startswith('0x'): + return int(num_str, 16) + elif num_str.startswith('0'): + return int(num_str, 8) + else: + return int(num_str) + + def _deduce_dev_name(self, e): + dev_types = { + 'pci': self._deduce_dev_name_pci, + 'scsi': self._deduce_dev_name_scsi, + 'usb': self._deduce_dev_name_usb, + } + return dev_types[e.attrib['type']](e) + + def _deduce_dev_name_pci(self, e): + attrib = {} + for field in ('domain', 'bus', 'slot', 'function'): + attrib[field] = self._toint(e.source.address.attrib[field]) + return 'pci_%(domain)04x_%(bus)02x_%(slot)02x_%(function)x' % attrib + + def _deduce_dev_name_scsi(self, e): + attrib = {} + for field in ('bus', 'target', 'unit'): + attrib[field] = self._toint(e.source.address.attrib[field]) + attrib['host'] = self._toint( + e.source.adapter.attrib['name'][len('scsi_host'):]) + return 'scsi_%(host)d_%(bus)d_%(target)d_%(unit)d' % attrib + + def _deduce_dev_name_usb(self, e): + dev_names = DevicesModel(conn=self.conn).get_list(_cap='usb_device') + usb_infos = [DeviceModel(conn=self.conn).lookup(dev_name) + for dev_name in dev_names] + + unknown_dev = None + + try: + evendor = self._toint(e.source.vendor.attrib['id']) + eproduct = self._toint(e.source.product.attrib['id']) + except AttributeError: + evendor = 0 + eproduct = 0 + else: + unknown_dev = 'usb_vendor_%s_product_%s' % (evendor, eproduct) + + try: + ebus = self._toint(e.source.address.attrib['bus']) + edevice = self._toint(e.source.address.attrib['device']) + except AttributeError: + ebus = -1 + edevice = -1 + else: + unknown_dev = 'usb_bus_%s_device_%s' % (ebus, edevice) + + for usb_info in usb_infos: + ivendor = self._toint(usb_info['vendor']['id']) + iproduct = self._toint(usb_info['product']['id']) + if evendor == ivendor and eproduct == iproduct: + return usb_info['name'] + ibus = usb_info['bus'] + idevice = usb_info['device'] + if ebus == ibus and edevice == idevice: + return usb_info['name'] + return unknown_dev + + def _passthrough_device_validate(self, dev_name): + eligible_dev_names = \ + DevicesModel(conn=self.conn).get_list(_passthrough='true') + if dev_name not in eligible_dev_names: + raise InvalidParameter('KCHVMHDEV0002E', {'dev_name': dev_name}) + + def create(self, vmid, params): + dev_name = params['name'] + self._passthrough_device_validate(dev_name) + dev_info = DeviceModel(conn=self.conn).lookup(dev_name) + attach_device = { + 'pci': self._attach_pci_device, + 'scsi': self._attach_scsi_device, + 'usb_device': self._attach_usb_device, + }[dev_info['device_type']] + return attach_device(vmid, dev_info) + + def _get_pci_device_xml(self, dev_info): + if 'detach_driver' not in dev_info: + dev_info['detach_driver'] = 'kvm' + + xmlstr = ''' + <hostdev mode='subsystem' type='pci' managed='yes'> + <source> + <address domain='%(domain)s' bus='%(bus)s' slot='%(slot)s' + function='%(function)s'/> + </source> + <driver name='%(detach_driver)s'/> + </hostdev>''' % dev_info + return xmlstr + + @staticmethod + def _validate_pci_passthrough_env(): + if not glob.glob('/sys/kernel/iommu_groups/*'): + raise InvalidOperation("KCHVMHDEV0003E") + + # Enable virt_use_sysfs on RHEL6 and older distributions + # In recent Fedora, there is no virt_use_sysfs. + out, err, rc = run_command(['getsebool', 'virt_use_sysfs']) + if rc == 0 and out.rstrip('\n') != "virt_use_sysfs --> on": + out, err, rc = run_command(['setsebool', '-P', + 'virt_use_sysfs=on']) + if rc != 0: + kimchi_log.warning("Unable to turn on sebool virt_use_sysfs") + + def _attach_pci_device(self, vmid, dev_info): + self._validate_pci_passthrough_env() + + dom = VMModel.get_vm(vmid, self.conn) + # Due to libvirt limitation, we don't support live assigne device to + # vfio driver. + driver = ('vfio' if DOM_STATE_MAP[dom.info()[0]] == "shutoff" and + CapabilitiesModel().kernel_vfio else 'kvm') + + # Attach all PCI devices in the same IOMMU group + dev_model = DeviceModel(conn=self.conn) + affected_devs_model = PassthroughAffectedDevicesModel(conn=self.conn) + dev_infos = [dev_model.lookup(dev_name) for dev_name in + affected_devs_model.get_list(dev_info['name'])] + pci_infos = [dev_info] + [info for info in dev_infos + if info['device_type'] == 'pci'] + + device_flags = get_vm_config_flag(dom, mode='all') + + with RollbackContext() as rollback: + for pci_info in pci_infos: + pci_info['detach_driver'] = driver + xmlstr = self._get_pci_device_xml(pci_info) + try: + dom.attachDeviceFlags(xmlstr, device_flags) + except libvirt.libvirtError: + kimchi_log.error( + 'Failed to attach host device %s to VM %s: \n%s', + pci_info['name'], vmid, xmlstr) + raise + rollback.prependDefer(dom.detachDeviceFlags, + xmlstr, device_flags) + rollback.commitAll() + + return dev_info['name'] + + def _get_scsi_device_xml(self, dev_info): + xmlstr = ''' + <hostdev mode='subsystem' type='scsi' sgio='unfiltered'> + <source> + <adapter name='scsi_host%(host)s'/> + <address type='scsi' bus='%(bus)s' target='%(target)s' + unit='%(lun)s'/> + </source> + </hostdev>''' % dev_info + return xmlstr + + def _attach_scsi_device(self, vmid, dev_info): + xmlstr = self._get_scsi_device_xml(dev_info) + dom = VMModel.get_vm(vmid, self.conn) + dom.attachDeviceFlags(xmlstr, get_vm_config_flag(dom, mode='all')) + return dev_info['name'] + + def _get_usb_device_xml(self, dev_info): + xmlstr = ''' + <hostdev mode='subsystem' type='usb' managed='yes'> + <source startupPolicy='optional'> + <vendor id='%s'/> + <product id='%s'/> + <address bus='%s' device='%s'/> + </source> + </hostdev>''' % (dev_info['vendor']['id'], dev_info['product']['id'], + dev_info['bus'], dev_info['device']) + return xmlstr + + def _attach_usb_device(self, vmid, dev_info): + xmlstr = self._get_usb_device_xml(dev_info) + dom = VMModel.get_vm(vmid, self.conn) + dom.attachDeviceFlags(xmlstr, get_vm_config_flag(dom, mode='all')) + return dev_info['name'] + + +class VMHostDevModel(object): + def __init__(self, **kargs): + self.conn = kargs['conn'] + + def lookup(self, vmid, dev_name): + dom = VMModel.get_vm(vmid, self.conn) + xmlstr = dom.XMLDesc(0) + root = objectify.fromstring(xmlstr) + try: + hostdev = root.devices.hostdev + except AttributeError: + raise NotFoundError('KCHVMHDEV0001E', + {'vmid': vmid, 'dev_name': dev_name}) + + devsmodel = VMHostDevsModel(conn=self.conn) + + for e in hostdev: + deduced_name = devsmodel._deduce_dev_name(e) + if deduced_name == dev_name: + return {'name': dev_name, 'type': e.attrib['type']} + + raise NotFoundError('KCHVMHDEV0001E', + {'vmid': vmid, 'dev_name': dev_name}) + + def delete(self, vmid, dev_name): + dom = VMModel.get_vm(vmid, self.conn) + xmlstr = dom.XMLDesc(0) + root = objectify.fromstring(xmlstr) + pci_devs = [] + + try: + hostdev = root.devices.hostdev + except AttributeError: + raise NotFoundError('KCHVMHDEV0001E', + {'vmid': vmid, 'dev_name': dev_name}) + + devsmodel = VMHostDevsModel(conn=self.conn) + + for e in hostdev: + deduced_name = devsmodel._deduce_dev_name(e) + if e.attrib['type'] == 'pci': + pci_devs.append((deduced_name, e)) + if deduced_name == dev_name: + dev_e = e + xmlstr = etree.tostring(e) + dom.detachDeviceFlags( + xmlstr, get_vm_config_flag(dom, mode='all')) + break + else: + raise NotFoundError('KCHVMHDEV0001E', + {'vmid': vmid, 'dev_name': dev_name}) + + if dev_e.attrib['type'] == 'pci': + self._delete_affected_pci_devices(dom, dev_name, pci_devs) + + def _delete_affected_pci_devices(self, dom, dev_name, pci_devs): + dev_model = DeviceModel(conn=self.conn) + try: + dev_model.lookup(dev_name) + except NotFoundError: + return + + affected_names = set( + PassthroughAffectedDevicesModel(conn=self.conn).get_list(dev_name)) + + for pci_name, e in pci_devs: + if pci_name in affected_names: + xmlstr = etree.tostring(e) + dom.detachDeviceFlags( + xmlstr, get_vm_config_flag(dom, mode='all')) diff --git a/src/kimchi/rollbackcontext.py b/src/kimchi/rollbackcontext.py index 70fcfeb..ba28999 100644 --- a/src/kimchi/rollbackcontext.py +++ b/src/kimchi/rollbackcontext.py @@ -64,3 +64,6 @@ class RollbackContext(object): def prependDefer(self, func, *args, **kwargs): self._finally.insert(0, (func, args, kwargs)) + + def commitAll(self): + self._finally = [] -- 1.9.3