[PATCH] [Kimchi v2 0/2] Set memory when attaching GPU
v2: - ignore iommu validation for testing gpu attachment. Set the window size accordingly when attaching a GPU. Jose Ricardo Ziviani (2): Set mmio memory when GPU is attached Create test cases for GPU attachment i18n.py | 2 +- mockmodel.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++- model/host.py | 4 +- model/vmhostdevs.py | 141 +++++++++++++++++++++++++++++++++++++++-- tests/test_mockmodel.py | 33 ++++++++++ 5 files changed, 336 insertions(+), 8 deletions(-) -- 1.9.1
Signed-off-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com> --- i18n.py | 2 +- model/host.py | 4 +- model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 140 insertions(+), 7 deletions(-) diff --git a/i18n.py b/i18n.py index a953b55..008e327 100644 --- a/i18n.py +++ b/i18n.py @@ -144,7 +144,7 @@ messages = { "For AMD CPU, add 'iommu=pt iommu=1'."), "KCHVMHDEV0004E": _('"name" should be a device name string'), "KCHVMHDEV0005E": _('The device %(name)s is probably in use by the host. Unable to attach it to the guest.'), - "KCHVMHDEV0006E": _('Hot-plug of device %(name)s is not supported.'), + "KCHVMHDEV0006E": _('Hot-(un)plug of device %(name)s is not supported.'), "KCHVMIF0001E": _("Interface %(iface)s does not exist in virtual machine %(name)s"), "KCHVMIF0002E": _("Network %(network)s specified for virtual machine %(name)s does not exist"), diff --git a/model/host.py b/model/host.py index 135c8a9..abf1191 100644 --- a/model/host.py +++ b/model/host.py @@ -146,9 +146,9 @@ class DevicesModel(object): class DeviceModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] - self.iommu_groups = self._get_iommu_groups() + self.iommu_groups = self.get_iommu_groups() - def _get_iommu_groups(self): + def get_iommu_groups(self): iommu_groups = defaultdict(list) conn = self.conn diff --git a/model/vmhostdevs.py b/model/vmhostdevs.py index dabe8e4..2a62249 100644 --- a/model/vmhostdevs.py +++ b/model/vmhostdevs.py @@ -22,7 +22,7 @@ import libvirt import os import platform from lxml import etree, objectify -from lxml.builder import E +from lxml.builder import E, ElementMaker from operator import itemgetter from wok.exception import InvalidOperation, InvalidParameter, NotFoundError @@ -34,6 +34,12 @@ from wok.plugins.kimchi.model.config import CapabilitiesModel from wok.plugins.kimchi.model.host import DeviceModel, DevicesModel from wok.plugins.kimchi.model.utils import get_vm_config_flag from wok.plugins.kimchi.model.vms import DOM_STATE_MAP, VMModel +from wok.plugins.kimchi.xmlutils.qemucmdline import get_qemucmdline_xml +from wok.plugins.kimchi.xmlutils.qemucmdline import QEMU_NAMESPACE + + +CMDLINE_FIELD_NAME = 'spapr-pci-host-bridge.mem_win_size' +WINDOW_SIZE_BAR = 0x800000000 class VMHostDevsModel(object): @@ -149,6 +155,7 @@ class VMHostDevsModel(object): slots = sorted(slots) + free = 0 for free, slot in enumerate(slots, start=1): if free < slot: return free @@ -184,8 +191,9 @@ class VMHostDevsModel(object): DOM_STATE_MAP[dom.info()[0]] == "shutoff" pci_infos = sorted(pci_infos, key=itemgetter('name')) - if dev_model.is_device_3D_controller(dev_info) and \ - DOM_STATE_MAP[dom.info()[0]] != "shutoff": + # does not allow hot-plug of 3D graphic cards + is_3D_device = dev_model.is_device_3D_controller(dev_info) + if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff": raise InvalidOperation('KCHVMHDEV0006E', {'name': dev_info['name']}) @@ -207,6 +215,12 @@ class VMHostDevsModel(object): device_flags = get_vm_config_flag(dom, mode='all') + # when attaching a 3D graphic device it might be necessary to increase + # the window size memory in order to be able to attach more than one + # device to the same guest + if is_3D_device: + self.update_mmio_guest(vmid, True) + slot = 0 if is_multifunction: slot = self._available_slot(dom) @@ -229,6 +243,115 @@ class VMHostDevsModel(object): return dev_info['name'] + def _count_3D_devices_attached(self, dom): + counter = 0 + root = objectify.fromstring(dom.XMLDesc(0)) + + try: + hostdev = root.devices.hostdev + + except AttributeError: + return counter + + for device in hostdev: + if device.attrib['type'] != 'pci': + continue + + name = DeviceModel.deduce_dev_name(device, self.conn) + info = DeviceModel(conn=self.conn).lookup(name) + if 'vga3d' in info and info['vga3d']: + counter += 1 + + return counter + + def update_mmio_guest(self, vmid, is_attaching): + dom = VMModel.get_vm(vmid, self.conn) + # get the number of 3D graphic cards already attached to the guest + # based on this number we will decide if the memory size will be + # increased or not + counter = self._count_3D_devices_attached(dom) + if counter == 0 and is_attaching: + return + + size = 0 + if is_attaching: + # suppose this is the 3rd graphic card to be attached to the same + # guest, counter will be 2+1 (2 existing + this attachment) times + # 32G (0x80000000) + size = hex((counter + 1) * WINDOW_SIZE_BAR) + + else: + size = hex(counter * WINDOW_SIZE_BAR) + + # if the guest already has the xml file we will simply update the + # value, otherwise we will add the new field + new_xml = self._update_win_memory_size(dom, counter, size) + if new_xml is None and is_attaching: + new_xml = self._add_win_memory_size(dom, size) + + # update the XML + self.conn.get().defineXML(new_xml) + + def _update_win_memory_size(self, dom, counter, wnd_size): + root = objectify.fromstring(dom.XMLDesc(0)) + + # look for the existing argument in <qemu:commandline> and try + # to update the value (or remove if there is only one (or none) + # graphic card attached. + cmdline = root.findall('{%s}commandline' % QEMU_NAMESPACE) + for line in cmdline: + for arg in line.iterchildren(): + if not arg.values()[0].startswith(CMDLINE_FIELD_NAME): + continue + + # update mem_win_size value + if counter > 1: + arg.set('value', CMDLINE_FIELD_NAME + '=' + wnd_size) + + # remove mem_win_size + elif counter <= 1: + line.remove(arg) + + return etree.tostring(root, encoding='utf-8', + pretty_print=True) + + return None + + def _add_win_memory_size(self, dom, wnd_size): + root = objectify.fromstring(dom.XMLDesc(0)) + val = CMDLINE_FIELD_NAME + '=' + wnd_size + + cmdline = root.find('{%s}commandline' % QEMU_NAMESPACE) + # <qemu:commandline> doesn't exist, create the full commandline xml + # with the required values and return + if cmdline is None: + args = {} + args['-global'] = val + root.append(etree.fromstring(get_qemucmdline_xml(args))) + return etree.tostring(root, encoding='utf-8', pretty_print=True) + + # <qemu:commandline> exists and already has the tag + # <qemu:arg value='-global'> (user could already been using this for + # something else), so we just add our <qemu:arg...> missing. + found = False + for arg in cmdline.iterchildren(): + if arg.values()[0] == '-global': + EM = ElementMaker(namespace=QEMU_NAMESPACE, + nsmap={'qemu': QEMU_NAMESPACE}) + cmdline.append(EM.arg(value=val)) + found = True + break + + # <qemu:commandline> exists but there is no <qemu:arg value global> + # so, we add those missing arguments inside the exising cmdline + if not found: + EM = ElementMaker(namespace=QEMU_NAMESPACE, + nsmap={'qemu': QEMU_NAMESPACE}) + cmdline.append(EM.arg(value='-global')) + cmdline.append(EM.arg(value=val)) + + return etree.tostring(root, encoding='utf-8', pretty_print=True) + def _get_scsi_device_xml(self, dev_info): adapter = E.adapter(name=('scsi_host%s' % dev_info['host'])) address = E.address(type='scsi', bus=str(dev_info['bus']), @@ -285,7 +408,8 @@ class VMHostDevModel(object): 'type': e.attrib['type'], 'product': dev_info.get('product', None), 'vendor': dev_info.get('vendor', None), - 'multifunction': dev_info.get('multifunction', None)} + 'multifunction': dev_info.get('multifunction', None), + 'vga3d': dev_info.get('vga3d', None)} raise NotFoundError('KCHVMHDEV0001E', {'vmid': vmid, 'dev_name': dev_name}) @@ -304,6 +428,13 @@ class VMHostDevModel(object): pci_devs = [(DeviceModel.deduce_dev_name(e, self.conn), e) for e in hostdev if e.attrib['type'] == 'pci'] + dev_model = DeviceModel(conn=self.conn) + dev_info = dev_model.lookup(dev_name) + is_3D_device = dev_model.is_device_3D_controller(dev_info) + if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff": + raise InvalidOperation('KCHVMHDEV0006E', + {'name': dev_info['name']}) + for e in hostdev: if DeviceModel.deduce_dev_name(e, self.conn) == dev_name: xmlstr = etree.tostring(e) @@ -311,6 +442,8 @@ class VMHostDevModel(object): xmlstr, get_vm_config_flag(dom, mode='all')) if e.attrib['type'] == 'pci': self._delete_affected_pci_devices(dom, dev_name, pci_devs) + if is_3D_device: + self.update_mmio_guest(vmid, False) break else: raise NotFoundError('KCHVMHDEV0001E', -- 1.9.1
Signed-off-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com> --- mockmodel.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++- tests/test_mockmodel.py | 33 ++++++++++ 2 files changed, 196 insertions(+), 1 deletion(-) diff --git a/mockmodel.py b/mockmodel.py index d010df7..a765770 100644 --- a/mockmodel.py +++ b/mockmodel.py @@ -35,7 +35,9 @@ from wok.plugins.kimchi import imageinfo from wok.plugins.kimchi import osinfo from wok.plugins.kimchi.model import cpuinfo from wok.plugins.kimchi.model import vmifaces +from wok.plugins.kimchi.model.groups import PAMGroupsModel from wok.plugins.kimchi.model.host import DeviceModel +from wok.plugins.kimchi.model.host import DevicesModel from wok.plugins.kimchi.model.libvirtstoragepool import IscsiPoolDef from wok.plugins.kimchi.model.libvirtstoragepool import NetfsPoolDef from wok.plugins.kimchi.model.libvirtstoragepool import StoragePoolDef @@ -47,7 +49,7 @@ from wok.plugins.kimchi.model.storagevolumes import StorageVolumesModel from wok.plugins.kimchi.model import storagevolumes from wok.plugins.kimchi.model.templates import LibvirtVMTemplate from wok.plugins.kimchi.model.users import PAMUsersModel -from wok.plugins.kimchi.model.groups import PAMGroupsModel +from wok.plugins.kimchi.model.vmhostdevs import VMHostDevsModel from wok.plugins.kimchi.utils import pool_name_from_uri from wok.plugins.kimchi.vmtemplate import VMTemplate @@ -118,6 +120,10 @@ class MockModel(Model): setattr(self, m, mock_method) DeviceModel.lookup = self._mock_device_lookup + DeviceModel.get_iommu_groups = self._mock_device_get_iommu_groups + DeviceModel.is_device_3D_controller = \ + self._mock_device_is_device_3D_controller + DevicesModel.get_list = self._mock_devices_get_list StoragePoolsModel._check_lvm = self._check_lvm StoragePoolModel._update_lvm_disks = self._update_lvm_disks StorageVolumesModel.get_list = self._mock_storagevolumes_get_list @@ -125,6 +131,8 @@ class MockModel(Model): LibvirtVMTemplate._get_volume_path = self._get_volume_path VMTemplate.get_iso_info = self._probe_image imageinfo.probe_image = self._probe_image + VMHostDevsModel._get_pci_device_xml = self._get_pci_device_xml + VMHostDevsModel._validate_pci_passthrough_env = self._validate_pci def reset(self): MockModel._mock_vms = defaultdict(list) @@ -326,6 +334,16 @@ class MockModel(Model): return [dev['name'] for dev in self._mock_devices.devices.values() if dev['device_type'] == _cap] + def _mock_device_get_iommu_groups(self): + return [dev['iommuGroup'] for dev in + self._mock_devices.devices.values() + if 'iommuGroup' in dev] + + def _mock_device_is_device_3D_controller(self, info): + if 'driver' in info and 'name' in info['driver']: + return info['driver']['name'] == 'nvidia' + return False + def _mock_device_lookup(self, dev_name): return self._mock_devices.devices[dev_name] @@ -405,6 +423,46 @@ class MockModel(Model): if sn.name == name: sn.current = True + def _attach_device(self, vm_name, xmlstr): + MockModel._mock_vms[vm_name].append(xmlstr) + + def _validate_pci(self): + pass + + def _get_pci_device_xml(self, dev_info, slot, is_multifunction): + if 'detach_driver' not in dev_info: + dev_info['detach_driver'] = 'kvm' + + source = E.source(E.address(domain=str(dev_info['domain']), + bus=str(dev_info['bus']), + slot=str(dev_info['slot']), + function=str(dev_info['function']))) + driver = E.driver(name=dev_info['detach_driver']) + + if is_multifunction: + multi = E.address(type='pci', + domain='0', + bus='0', + slot=str(slot), + function=str(dev_info['function'])) + + if dev_info['function'] == 0: + multi = E.address(type='pci', + domain='0', + bus='0', + slot=str(slot), + function=str(dev_info['function']), + multifunction='on') + + host_dev = E.hostdev(source, driver, multi, + mode='subsystem', type='pci', managed='yes') + + else: + host_dev = E.hostdev(source, driver, + mode='subsystem', type='pci', managed='yes') + + return ET.tostring(host_dev) + class MockStorageVolumes(object): def __init__(self): @@ -481,6 +539,7 @@ class MockDevices(object): 'Centrino Advanced-N 6205 [Taylor Peak]', 'id': '0x0085'}, 'slot': 0, + 'vga3d': False, 'vendor': {'description': 'Intel Corporation', 'id': '0x8086'}}, 'pci_0000_0d_00_0': {'bus': 13, @@ -497,8 +556,111 @@ class MockDevices(object): 'PCIe SDXC/MMC Host Controller', 'id': '0xe823'}, 'slot': 0, + 'vga3d': False, 'vendor': {'description': 'Ricoh Co Ltd', 'id': '0x1180'}}, + 'pci_0000_09_01_0': {'bus': 9, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'tg3'}, + 'function': 0, + 'iommuGroup': 8, + 'name': 'pci_0000_09_01_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:09:01.0', + 'product': {'description': + 'NetXtreme BCM5719', + 'id': '0x1657'}, + 'slot': 1, + 'vga3d': False, + 'vendor': {'description': 'Broadcom Corp', + 'id': '0x14e4'}}, + 'pci_0000_09_01_1': {'bus': 9, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'tg3'}, + 'function': 1, + 'iommuGroup': 8, + 'name': 'pci_0000_09_01_1', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:09:01.1', + 'product': {'description': + 'NetXtreme BCM5719', + 'id': '0x1657'}, + 'slot': 1, + 'vga3d': False, + 'vendor': {'description': 'Broadcom Corp', + 'id': '0x14e4'}}, + 'pci_0000_09_01_2': {'bus': 9, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'tg3'}, + 'function': 2, + 'iommuGroup': 8, + 'name': 'pci_0000_09_01_2', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:09:01.2', + 'product': {'description': + 'NetXtreme BCM5719', + 'id': '0x1657'}, + 'slot': 1, + 'vga3d': False, + 'vendor': {'description': 'Broadcom Corp', + 'id': '0x14e4'}}, + 'pci_0000_1a_00_0': {'bus': 26, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'nvidia'}, + 'function': 0, + 'iommuGroup': 9, + 'name': 'pci_0000_1a_00_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:1a:00.0', + 'product': {'description': + 'GK210GL [Tesla K80]', + 'id': '0x0302'}, + 'slot': 0, + 'vga3d': True, + 'vendor': {'description': 'NVIDIA Corp', + 'id': '0x10de'}}, + 'pci_0001_1b_00_0': {'bus': 27, + 'device_type': 'pci', + 'domain': 1, + 'driver': {'name': 'nvidia'}, + 'function': 0, + 'iommuGroup': 7, + 'name': 'pci_0001_1b_00_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0001:1b:00.0', + 'product': {'description': + 'GK210GL [Tesla K80]', + 'id': '0x0302'}, + 'slot': 0, + 'vga3d': True, + 'vendor': {'description': 'NVIDIA Corp', + 'id': '0x10de'}}, + 'pci_0000_1c_00_0': {'bus': 28, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'nvidia'}, + 'function': 0, + 'iommuGroup': 7, + 'name': 'pci_0000_1c_00_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:0d:00.0', + 'product': {'description': + 'GK210GL [Tesla K80]', + 'id': '0x0302'}, + 'slot': 0, + 'vga3d': True, + 'vendor': {'description': 'NVIDIA Corp', + 'id': '0x10de'}}, 'scsi_host0': {'adapter': {'fabric_wwn': '37df6c1efa1b4388', 'type': 'fc_host', 'wwnn': 'efb6563f06434a98', diff --git a/tests/test_mockmodel.py b/tests/test_mockmodel.py index 5c9fad4..0668191 100644 --- a/tests/test_mockmodel.py +++ b/tests/test_mockmodel.py @@ -26,6 +26,7 @@ import unittest from tests.utils import get_free_port, patch_auth, request, run_server from tests.utils import wait_task +from wok.exception import InvalidOperation from wok.plugins.kimchi import mockmodel from wok.plugins.kimchi.osinfo import get_template_default @@ -112,6 +113,38 @@ class MockModelTests(unittest.TestCase): vms.append(u'test') self.assertEqual(model.vms_get_list(), sorted(vms)) + def test_memory_window_changes(self): + model.templates_create({'name': u'test', + 'cdrom': fake_iso}) + task = model.vms_create({'name': u'test-vm', + 'template': '/plugins/kimchi/templates/test'}) + wait_task(model.task_lookup, task['id']) + + info = model.device_lookup('pci_0000_1a_00_0') + model.vmhostdevs_update_mmio_guest(u'test-vm', True) + model._attach_device(u'test-vm', + model._get_pci_device_xml(info, 0, False)) + + def test_hotplug_3D_card(self): + model.templates_create({'name': u'test', + 'cdrom': fake_iso}) + task = model.vms_create({'name': u'test-vm', + 'template': '/plugins/kimchi/templates/test'}) + wait_task(model.task_lookup, task['id']) + model.vm_start(u'test-vm') + + # attach the 3D cards found to a running guest + all_devices = model.devices_get_list() + for device in all_devices: + device_info = model.device_lookup(device) + if model.device_is_device_3D_controller(device_info): + try: + model.vmhostdevs_create(u'test-vm', {'name': device}) + + # expect the error: KCHVMHDEV0006E + except InvalidOperation as e: + self.assertEqual(e.message[:14], u'KCHVMHDEV0006E') + def test_vm_info(self): model.templates_create({'name': u'test', 'cdrom': fake_iso}) -- 1.9.1
Reviewed-By: Ramon Medeiros <ramonn@br.ibm.com> On 03/01/2016 02:12 PM, Jose Ricardo Ziviani wrote:
v2: - ignore iommu validation for testing gpu attachment.
Set the window size accordingly when attaching a GPU.
Jose Ricardo Ziviani (2): Set mmio memory when GPU is attached Create test cases for GPU attachment
i18n.py | 2 +- mockmodel.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++- model/host.py | 4 +- model/vmhostdevs.py | 141 +++++++++++++++++++++++++++++++++++++++-- tests/test_mockmodel.py | 33 ++++++++++ 5 files changed, 336 insertions(+), 8 deletions(-)
-- Ramon Nunes Medeiros Kimchi Developer Linux Technology Center Brazil IBM Systems & Technology Group Phone : +55 19 2132 7878 ramonn@br.ibm.com
participants (3)
-
Aline Manera -
Jose Ricardo Ziviani -
Ramon Medeiros