Re: [Kimchi-devel] [PATCH v11 3/6] Host device passthrough: Directly assign and dissmis host device from VM

On 09/30/2014 07:00 AM, Zhou Zheng Sheng wrote: > This patch enbales Kimchi's VM to use host devices directly, and it > greatly improves the related device performance. The user can assign > PCI, USB and SCSI LUN directly to VM, as long as the host supports one > of Intel VT-d, AMD IOMMU or POWER sPAPR technology and runs a recent > release of Linux kernel. > > This patch adds a sub-collection "hostdevs" to the URI vms/vm-name/. > The front-end can GET > vms/vm-name/hostdevs > and > vms/vm-name/hostdevs/dev-name > or POST (assign) > vms/vm-name/hostdevs > and DELETE (dismiss) > vms/vm-name/hostdevs/dev-name > > The eligible devices to assign are the devices listed by the URI > host/devices?_passthrough=1 > When assigning a host PCI device to VM, all the eligible PCI devices in > the same IOMMU group are also automatically assigned, and vice versa > when dismissing a host PIC device from the VM. > > Some examples: > > Assign a USB device: > curl -k -u root -H "Content-Type: application/json" \ > -H "Accept: application/json" \ > -X POST -d '{"name": "usb_1_1_6"}' \ > 'https://127.0.0.1:8001/vms/rhel65/hostdevs' > > Assign a PCI device: > -d '{"name": "pci_0000_0d_00_0"}' > > Assign a SCSI LUN: > -d '{"name": "scsi_1_0_0_0"}' > > List assigned devices: > curl -k -u root -H "Content-Type: application/json" \ > -H "Accept: application/json" \ > 'https://127.0.0.1:8001/vms/rhel65/hostdevs' > The above command should print following. > [ > { > "type":"scsi", > "name":"scsi_1_0_0_0" > }, > { > "type":"usb", > "name":"usb_1_1_6" > }, > { > "type":"pci", > "name":"pci_0000_0d_00_0" > }, > { > "type":"pci", > "name":"pci_0000_03_00_0" > } > ] > Notice that the device pci_0000_03_00_0 is also assigned automatically. > > The assigned devices are hot-plugged to VM and also written to the > domain XML. When it's possible, it enables VFIO for PCI device > assignment. > > On distribution with old Linux kernel, there are many limitations with > PCI passthrough and it's hardly useful. This patch tries to adapt to old > kernel but it's better to use a newer kernel with vfio support. Thus > this patch also provide a new capability in /config/capabilities. > > curl -k -u root -H "Content-Type: application/json" \ > -H "Accept: application/json" \ > 'https://127.0.0.1:8001/config/capabilities' > > The above command should print following. > { > "blah": "blah", > ... > "kernel_vfio":true > } Just to make clear for a UI perspective: when "kernel_vfio" is false, we should disable PCI passthrough on UI, right?

> v1: > Handle the devices in the VM template. > > v2: > Handle the devices in the VM sub-resource "hostdevs". > > v3: > No change. > > v4: > Not all domain XMLs contain hostdev node. Deal with the case. > > v5: > Change _passthrough='1' to _passthrough='true'. When attaching and > detaching a device, do not use VIR_DOMAIN_AFFECT_CURRENT flag, instead, > use kimchi.model.utils.get_vm_config_flag() to correctly set the device > flag. > > v11: > Add Capability kernel_vfio to indicate if Linux kernel is new enough > to support vfio. > > Signed-off-by: Zhou Zheng Sheng <zhshzhou(a)linux.vnet.ibm.com&gt; > --- > src/kimchi/control/vm/hostdevs.py | 44 ++++++ > src/kimchi/featuretests.py | 10 +- > src/kimchi/i18n.py | 7 + > src/kimchi/model/config.py | 6 +- > src/kimchi/model/vmhostdevs.py | 305 > ++++++++++++++++++++++++++++++++++++++ > src/kimchi/rollbackcontext.py | 3 + > 6 files changed, 373 insertions(+), 2 deletions(-) > create mode 100644 src/kimchi/control/vm/hostdevs.py > create mode 100644 src/kimchi/model/vmhostdevs.py > > diff --git a/src/kimchi/control/vm/hostdevs.py > b/src/kimchi/control/vm/hostdevs.py > new file mode 100644 > index 0000000..81fe8ec > --- /dev/null > +++ b/src/kimchi/control/vm/hostdevs.py > @@ -0,0 +1,44 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +from kimchi.control.base import Collection, Resource > +from kimchi.control.utils import UrlSubNode > + > + > +@UrlSubNode("hostdevs") > +class VMHostDevs(Collection): > + def __init__(self, model, vmid): > + super(VMHostDevs, self).__init__(model) > + self.resource = VMHostDev > + self.vmid = vmid > + self.resource_args = [self.vmid, ] > + self.model_args = [self.vmid, ] > + > + > +class VMHostDev(Resource): > + def __init__(self, model, vmid, ident): > + super(VMHostDev, self).__init__(model, ident) > + self.vmid = vmid > + self.ident = ident > + self.info = {} Why that is needed?

> diff --git a/src/kimchi/model/vmhostdevs.py > b/src/kimchi/model/vmhostdevs.py > new file mode 100644 > index 0000000..0d002b5 > --- /dev/null > +++ b/src/kimchi/model/vmhostdevs.py > @@ -0,0 +1,305 @@ > +# > +# Project Kimchi > +# > +# Copyright IBM, Corp. 2014 > +# > +# This library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > +# > +# This library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > +# > +# You should have received a copy of the GNU Lesser General Public > +# License along with this library; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > + > +import glob > +import os > + > +import libvirt > +from lxml import etree, objectify > + > +from kimchi.exception import InvalidOperation, InvalidParameter, > NotFoundError > +from kimchi.model.config import CapabilitiesModel > +from kimchi.model.host import DeviceModel, DevicesModel > +from kimchi.model.utils import get_vm_config_flag > +from kimchi.model.vms import DOM_STATE_MAP, VMModel > +from kimchi.rollbackcontext import RollbackContext > +from kimchi.utils import kimchi_log, run_command > + > + > +class VMHostDevsModel(object): > + def __init__(self, **kargs): > + self.conn = kargs['conn'] > + > + def get_list(self, vmid): > + dom = VMModel.get_vm(vmid, self.conn) > + xmlstr = dom.XMLDesc(0) > + root = objectify.fromstring(xmlstr) > + try: > + hostdev = root.devices.hostdev > + except AttributeError: > + return [] > + > + return [self._deduce_dev_name(e) for e in hostdev] > + > + @staticmethod > + def _toint(num_str): > + if num_str.startswith('0x'): > + return int(num_str, 16) > + elif num_str.startswith('0'): > + return int(num_str, 8) > + else: > + return int(num_str) > + > + def _deduce_dev_name(self, e): > + dev_types = { > + 'pci': self._deduce_dev_name_pci, > + 'scsi': self._deduce_dev_name_scsi, > + 'usb': self._deduce_dev_name_usb, > + } > + return dev_types[e.attrib['type']](e) > + To avoid having a map for it: return getattr(self, "_deduce_dev_name_%s" % type)(e)

> + attach_device = { > + 'pci': self._attach_pci_device, > + 'scsi': self._attach_scsi_device, > + 'usb_device': self._attach_usb_device, > + }[dev_info['device_type']] > + return attach_device(vmid, dev_info) > + Same here. return getattr(self, "_attach_%s_device" % dev_info['device_type'])(vmid, dev_info)

> + def _get_pci_device_xml(self, dev_info): > + if 'detach_driver' not in dev_info: > + dev_info['detach_driver'] = 'kvm' > + > + xmlstr = ''' > + <hostdev mode='subsystem' type='pci' managed='yes'> > + <source> > + <address domain='%(domain)s' bus='%(bus)s' slot='%(slot)s' > + function='%(function)s'/> > + </source> > + <driver name='%(detach_driver)s'/> > + </hostdev>''' % dev_info We are on a movement to use etree.builder to create the XML. It would be good to have it in that way too.

> + return xmlstr > + > + @staticmethod > + def _validate_pci_passthrough_env(): > + # Linux kernel < 3.5 doesn't provide /sys/kernel/iommu_groups > + if os.path.isdir('/sys/kernel/iommu_groups'): > + if not glob.glob('/sys/kernel/iommu_groups/*'): > + raise InvalidOperation("KCHVMHDEV0003E") > + > + # Enable virt_use_sysfs on RHEL6 and older distributions > + # In recent Fedora, there is no virt_use_sysfs. > + out, err, rc = run_command(['getsebool', 'virt_use_sysfs']) > + if rc == 0 and out.rstrip('\n') != "virt_use_sysfs --> on": > + out, err, rc = run_command(['setsebool', '-P', > + 'virt_use_sysfs=on']) > + if rc != 0: > + kimchi_log.warning("Unable to turn on sebool > virt_use_sysfs") > + > + def _attach_pci_device(self, vmid, dev_info): > + self._validate_pci_passthrough_env() > + > + dom = VMModel.get_vm(vmid, self.conn) > + # Due to libvirt limitation, we don't support live assigne > device to > + # vfio driver. > + driver = ('vfio' if DOM_STATE_MAP[dom.info()[0]] == "shutoff" > and > + CapabilitiesModel().kernel_vfio else 'kvm') > + > + # Attach all PCI devices in the same IOMMU group > + dev_model = DeviceModel(conn=self.conn) > + devs_model = DevicesModel(conn=self.conn) > + affected_names = devs_model.get_list( > + _passthrough_affected_by=dev_info['name']) > + passthrough_names = devs_model.get_list( > + _cap='pci', _passthrough='true') > + group_names = list(set(affected_names) & set(passthrough_names)) > + pci_infos = [dev_model.lookup(dev_name) for dev_name in > group_names] > + pci_infos.append(dev_info) > + > + device_flags = get_vm_config_flag(dom, mode='all') > + > + with RollbackContext() as rollback: > + for pci_info in pci_infos: > + pci_info['detach_driver'] = driver > + xmlstr = self._get_pci_device_xml(pci_info) > + try: > + dom.attachDeviceFlags(xmlstr, device_flags) > + except libvirt.libvirtError: > + kimchi_log.error( > + 'Failed to attach host device %s to VM %s: > \n%s', > + pci_info['name'], vmid, xmlstr) > + raise > + rollback.prependDefer(dom.detachDeviceFlags, > + xmlstr, device_flags) > + rollback.commitAll() > + > + return dev_info['name'] > + > + def _get_scsi_device_xml(self, dev_info): > + xmlstr = ''' > + <hostdev mode='subsystem' type='scsi' sgio='unfiltered'> > + <source> > + <adapter name='scsi_host%(host)s'/> > + <address type='scsi' bus='%(bus)s' target='%(target)s' > + unit='%(lun)s'/> > + </source> > + </hostdev>''' % dev_info Same here about etree.builder > + return xmlstr > + > + def _attach_scsi_device(self, vmid, dev_info): > + xmlstr = self._get_scsi_device_xml(dev_info) > + dom = VMModel.get_vm(vmid, self.conn) > + dom.attachDeviceFlags(xmlstr, get_vm_config_flag(dom, > mode='all')) > + return dev_info['name'] > + > + def _get_usb_device_xml(self, dev_info): > + xmlstr = ''' > + <hostdev mode='subsystem' type='usb' managed='yes'> > + <source startupPolicy='optional'> > + <vendor id='%s'/> > + <product id='%s'/> > + <address bus='%s' device='%s'/> > + </source> > + </hostdev>''' % (dev_info['vendor']['id'], > dev_info['product']['id'], > + dev_info['bus'], dev_info['device']) And all other related XML creation =)