[PATCH] [Kimchi 0/2] Set memory when attaching GPU
Set the window size accordingly when attaching a GPU. Jose Ricardo Ziviani (2): Set mmio memory when GPU is attached Create test cases for GPU attachment i18n.py | 2 +- mockmodel.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++- model/host.py | 4 +- model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++-- tests/test_mockmodel.py | 33 ++++++++++ 5 files changed, 332 insertions(+), 8 deletions(-) -- 1.9.1
Signed-off-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com> --- i18n.py | 2 +- model/host.py | 4 +- model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 140 insertions(+), 7 deletions(-) diff --git a/i18n.py b/i18n.py index a953b55..008e327 100644 --- a/i18n.py +++ b/i18n.py @@ -144,7 +144,7 @@ messages = { "For AMD CPU, add 'iommu=pt iommu=1'."), "KCHVMHDEV0004E": _('"name" should be a device name string'), "KCHVMHDEV0005E": _('The device %(name)s is probably in use by the host. Unable to attach it to the guest.'), - "KCHVMHDEV0006E": _('Hot-plug of device %(name)s is not supported.'), + "KCHVMHDEV0006E": _('Hot-(un)plug of device %(name)s is not supported.'), "KCHVMIF0001E": _("Interface %(iface)s does not exist in virtual machine %(name)s"), "KCHVMIF0002E": _("Network %(network)s specified for virtual machine %(name)s does not exist"), diff --git a/model/host.py b/model/host.py index 135c8a9..abf1191 100644 --- a/model/host.py +++ b/model/host.py @@ -146,9 +146,9 @@ class DevicesModel(object): class DeviceModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] - self.iommu_groups = self._get_iommu_groups() + self.iommu_groups = self.get_iommu_groups() - def _get_iommu_groups(self): + def get_iommu_groups(self): iommu_groups = defaultdict(list) conn = self.conn diff --git a/model/vmhostdevs.py b/model/vmhostdevs.py index dabe8e4..2a62249 100644 --- a/model/vmhostdevs.py +++ b/model/vmhostdevs.py @@ -22,7 +22,7 @@ import libvirt import os import platform from lxml import etree, objectify -from lxml.builder import E +from lxml.builder import E, ElementMaker from operator import itemgetter from wok.exception import InvalidOperation, InvalidParameter, NotFoundError @@ -34,6 +34,12 @@ from wok.plugins.kimchi.model.config import CapabilitiesModel from wok.plugins.kimchi.model.host import DeviceModel, DevicesModel from wok.plugins.kimchi.model.utils import get_vm_config_flag from wok.plugins.kimchi.model.vms import DOM_STATE_MAP, VMModel +from wok.plugins.kimchi.xmlutils.qemucmdline import get_qemucmdline_xml +from wok.plugins.kimchi.xmlutils.qemucmdline import QEMU_NAMESPACE + + +CMDLINE_FIELD_NAME = 'spapr-pci-host-bridge.mem_win_size' +WINDOW_SIZE_BAR = 0x800000000 class VMHostDevsModel(object): @@ -149,6 +155,7 @@ class VMHostDevsModel(object): slots = sorted(slots) + free = 0 for free, slot in enumerate(slots, start=1): if free < slot: return free @@ -184,8 +191,9 @@ class VMHostDevsModel(object): DOM_STATE_MAP[dom.info()[0]] == "shutoff" pci_infos = sorted(pci_infos, key=itemgetter('name')) - if dev_model.is_device_3D_controller(dev_info) and \ - DOM_STATE_MAP[dom.info()[0]] != "shutoff": + # does not allow hot-plug of 3D graphic cards + is_3D_device = dev_model.is_device_3D_controller(dev_info) + if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff": raise InvalidOperation('KCHVMHDEV0006E', {'name': dev_info['name']}) @@ -207,6 +215,12 @@ class VMHostDevsModel(object): device_flags = get_vm_config_flag(dom, mode='all') + # when attaching a 3D graphic device it might be necessary to increase + # the window size memory in order to be able to attach more than one + # device to the same guest + if is_3D_device: + self.update_mmio_guest(vmid, True) + slot = 0 if is_multifunction: slot = self._available_slot(dom) @@ -229,6 +243,115 @@ class VMHostDevsModel(object): return dev_info['name'] + def _count_3D_devices_attached(self, dom): + counter = 0 + root = objectify.fromstring(dom.XMLDesc(0)) + + try: + hostdev = root.devices.hostdev + + except AttributeError: + return counter + + for device in hostdev: + if device.attrib['type'] != 'pci': + continue + + name = DeviceModel.deduce_dev_name(device, self.conn) + info = DeviceModel(conn=self.conn).lookup(name) + if 'vga3d' in info and info['vga3d']: + counter += 1 + + return counter + + def update_mmio_guest(self, vmid, is_attaching): + dom = VMModel.get_vm(vmid, self.conn) + # get the number of 3D graphic cards already attached to the guest + # based on this number we will decide if the memory size will be + # increased or not + counter = self._count_3D_devices_attached(dom) + if counter == 0 and is_attaching: + return + + size = 0 + if is_attaching: + # suppose this is the 3rd graphic card to be attached to the same + # guest, counter will be 2+1 (2 existing + this attachment) times + # 32G (0x80000000) + size = hex((counter + 1) * WINDOW_SIZE_BAR) + + else: + size = hex(counter * WINDOW_SIZE_BAR) + + # if the guest already has the xml file we will simply update the + # value, otherwise we will add the new field + new_xml = self._update_win_memory_size(dom, counter, size) + if new_xml is None and is_attaching: + new_xml = self._add_win_memory_size(dom, size) + + # update the XML + self.conn.get().defineXML(new_xml) + + def _update_win_memory_size(self, dom, counter, wnd_size): + root = objectify.fromstring(dom.XMLDesc(0)) + + # look for the existing argument in <qemu:commandline> and try + # to update the value (or remove if there is only one (or none) + # graphic card attached. + cmdline = root.findall('{%s}commandline' % QEMU_NAMESPACE) + for line in cmdline: + for arg in line.iterchildren(): + if not arg.values()[0].startswith(CMDLINE_FIELD_NAME): + continue + + # update mem_win_size value + if counter > 1: + arg.set('value', CMDLINE_FIELD_NAME + '=' + wnd_size) + + # remove mem_win_size + elif counter <= 1: + line.remove(arg) + + return etree.tostring(root, encoding='utf-8', + pretty_print=True) + + return None + + def _add_win_memory_size(self, dom, wnd_size): + root = objectify.fromstring(dom.XMLDesc(0)) + val = CMDLINE_FIELD_NAME + '=' + wnd_size + + cmdline = root.find('{%s}commandline' % QEMU_NAMESPACE) + # <qemu:commandline> doesn't exist, create the full commandline xml + # with the required values and return + if cmdline is None: + args = {} + args['-global'] = val + root.append(etree.fromstring(get_qemucmdline_xml(args))) + return etree.tostring(root, encoding='utf-8', pretty_print=True) + + # <qemu:commandline> exists and already has the tag + # <qemu:arg value='-global'> (user could already been using this for + # something else), so we just add our <qemu:arg...> missing. + found = False + for arg in cmdline.iterchildren(): + if arg.values()[0] == '-global': + EM = ElementMaker(namespace=QEMU_NAMESPACE, + nsmap={'qemu': QEMU_NAMESPACE}) + cmdline.append(EM.arg(value=val)) + found = True + break + + # <qemu:commandline> exists but there is no <qemu:arg value global> + # so, we add those missing arguments inside the exising cmdline + if not found: + EM = ElementMaker(namespace=QEMU_NAMESPACE, + nsmap={'qemu': QEMU_NAMESPACE}) + cmdline.append(EM.arg(value='-global')) + cmdline.append(EM.arg(value=val)) + + return etree.tostring(root, encoding='utf-8', pretty_print=True) + def _get_scsi_device_xml(self, dev_info): adapter = E.adapter(name=('scsi_host%s' % dev_info['host'])) address = E.address(type='scsi', bus=str(dev_info['bus']), @@ -285,7 +408,8 @@ class VMHostDevModel(object): 'type': e.attrib['type'], 'product': dev_info.get('product', None), 'vendor': dev_info.get('vendor', None), - 'multifunction': dev_info.get('multifunction', None)} + 'multifunction': dev_info.get('multifunction', None), + 'vga3d': dev_info.get('vga3d', None)} raise NotFoundError('KCHVMHDEV0001E', {'vmid': vmid, 'dev_name': dev_name}) @@ -304,6 +428,13 @@ class VMHostDevModel(object): pci_devs = [(DeviceModel.deduce_dev_name(e, self.conn), e) for e in hostdev if e.attrib['type'] == 'pci'] + dev_model = DeviceModel(conn=self.conn) + dev_info = dev_model.lookup(dev_name) + is_3D_device = dev_model.is_device_3D_controller(dev_info) + if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff": + raise InvalidOperation('KCHVMHDEV0006E', + {'name': dev_info['name']}) + for e in hostdev: if DeviceModel.deduce_dev_name(e, self.conn) == dev_name: xmlstr = etree.tostring(e) @@ -311,6 +442,8 @@ class VMHostDevModel(object): xmlstr, get_vm_config_flag(dom, mode='all')) if e.attrib['type'] == 'pci': self._delete_affected_pci_devices(dom, dev_name, pci_devs) + if is_3D_device: + self.update_mmio_guest(vmid, False) break else: raise NotFoundError('KCHVMHDEV0001E', -- 1.9.1
Signed-off-by: Jose Ricardo Ziviani <joserz@linux.vnet.ibm.com> --- mockmodel.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++- tests/test_mockmodel.py | 33 ++++++++++ 2 files changed, 192 insertions(+), 1 deletion(-) diff --git a/mockmodel.py b/mockmodel.py index d010df7..034e6f6 100644 --- a/mockmodel.py +++ b/mockmodel.py @@ -35,7 +35,9 @@ from wok.plugins.kimchi import imageinfo from wok.plugins.kimchi import osinfo from wok.plugins.kimchi.model import cpuinfo from wok.plugins.kimchi.model import vmifaces +from wok.plugins.kimchi.model.groups import PAMGroupsModel from wok.plugins.kimchi.model.host import DeviceModel +from wok.plugins.kimchi.model.host import DevicesModel from wok.plugins.kimchi.model.libvirtstoragepool import IscsiPoolDef from wok.plugins.kimchi.model.libvirtstoragepool import NetfsPoolDef from wok.plugins.kimchi.model.libvirtstoragepool import StoragePoolDef @@ -47,7 +49,7 @@ from wok.plugins.kimchi.model.storagevolumes import StorageVolumesModel from wok.plugins.kimchi.model import storagevolumes from wok.plugins.kimchi.model.templates import LibvirtVMTemplate from wok.plugins.kimchi.model.users import PAMUsersModel -from wok.plugins.kimchi.model.groups import PAMGroupsModel +from wok.plugins.kimchi.model.vmhostdevs import VMHostDevsModel from wok.plugins.kimchi.utils import pool_name_from_uri from wok.plugins.kimchi.vmtemplate import VMTemplate @@ -118,6 +120,10 @@ class MockModel(Model): setattr(self, m, mock_method) DeviceModel.lookup = self._mock_device_lookup + DeviceModel.get_iommu_groups = self._mock_device_get_iommu_groups + DeviceModel.is_device_3D_controller = \ + self._mock_device_is_device_3D_controller + DevicesModel.get_list = self._mock_devices_get_list StoragePoolsModel._check_lvm = self._check_lvm StoragePoolModel._update_lvm_disks = self._update_lvm_disks StorageVolumesModel.get_list = self._mock_storagevolumes_get_list @@ -125,6 +131,7 @@ class MockModel(Model): LibvirtVMTemplate._get_volume_path = self._get_volume_path VMTemplate.get_iso_info = self._probe_image imageinfo.probe_image = self._probe_image + VMHostDevsModel._get_pci_device_xml = self._get_pci_device_xml def reset(self): MockModel._mock_vms = defaultdict(list) @@ -326,6 +333,16 @@ class MockModel(Model): return [dev['name'] for dev in self._mock_devices.devices.values() if dev['device_type'] == _cap] + def _mock_device_get_iommu_groups(self): + return [dev['iommuGroup'] for dev in + self._mock_devices.devices.values() + if 'iommuGroup' in dev] + + def _mock_device_is_device_3D_controller(self, info): + if 'driver' in info and 'name' in info['driver']: + return info['driver']['name'] == 'nvidia' + return False + def _mock_device_lookup(self, dev_name): return self._mock_devices.devices[dev_name] @@ -405,6 +422,43 @@ class MockModel(Model): if sn.name == name: sn.current = True + def _attach_device(self, vm_name, xmlstr): + MockModel._mock_vms[vm_name].append(xmlstr) + + def _get_pci_device_xml(self, dev_info, slot, is_multifunction): + if 'detach_driver' not in dev_info: + dev_info['detach_driver'] = 'kvm' + + source = E.source(E.address(domain=str(dev_info['domain']), + bus=str(dev_info['bus']), + slot=str(dev_info['slot']), + function=str(dev_info['function']))) + driver = E.driver(name=dev_info['detach_driver']) + + if is_multifunction: + multi = E.address(type='pci', + domain='0', + bus='0', + slot=str(slot), + function=str(dev_info['function'])) + + if dev_info['function'] == 0: + multi = E.address(type='pci', + domain='0', + bus='0', + slot=str(slot), + function=str(dev_info['function']), + multifunction='on') + + host_dev = E.hostdev(source, driver, multi, + mode='subsystem', type='pci', managed='yes') + + else: + host_dev = E.hostdev(source, driver, + mode='subsystem', type='pci', managed='yes') + + return ET.tostring(host_dev) + class MockStorageVolumes(object): def __init__(self): @@ -481,6 +535,7 @@ class MockDevices(object): 'Centrino Advanced-N 6205 [Taylor Peak]', 'id': '0x0085'}, 'slot': 0, + 'vga3d': False, 'vendor': {'description': 'Intel Corporation', 'id': '0x8086'}}, 'pci_0000_0d_00_0': {'bus': 13, @@ -497,8 +552,111 @@ class MockDevices(object): 'PCIe SDXC/MMC Host Controller', 'id': '0xe823'}, 'slot': 0, + 'vga3d': False, 'vendor': {'description': 'Ricoh Co Ltd', 'id': '0x1180'}}, + 'pci_0000_09_01_0': {'bus': 9, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'tg3'}, + 'function': 0, + 'iommuGroup': 8, + 'name': 'pci_0000_09_01_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:09:01.0', + 'product': {'description': + 'NetXtreme BCM5719', + 'id': '0x1657'}, + 'slot': 1, + 'vga3d': False, + 'vendor': {'description': 'Broadcom Corp', + 'id': '0x14e4'}}, + 'pci_0000_09_01_1': {'bus': 9, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'tg3'}, + 'function': 1, + 'iommuGroup': 8, + 'name': 'pci_0000_09_01_1', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:09:01.1', + 'product': {'description': + 'NetXtreme BCM5719', + 'id': '0x1657'}, + 'slot': 1, + 'vga3d': False, + 'vendor': {'description': 'Broadcom Corp', + 'id': '0x14e4'}}, + 'pci_0000_09_01_2': {'bus': 9, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'tg3'}, + 'function': 2, + 'iommuGroup': 8, + 'name': 'pci_0000_09_01_2', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:09:01.2', + 'product': {'description': + 'NetXtreme BCM5719', + 'id': '0x1657'}, + 'slot': 1, + 'vga3d': False, + 'vendor': {'description': 'Broadcom Corp', + 'id': '0x14e4'}}, + 'pci_0000_1a_00_0': {'bus': 26, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'nvidia'}, + 'function': 0, + 'iommuGroup': 9, + 'name': 'pci_0000_1a_00_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:1a:00.0', + 'product': {'description': + 'GK210GL [Tesla K80]', + 'id': '0x0302'}, + 'slot': 0, + 'vga3d': True, + 'vendor': {'description': 'NVIDIA Corp', + 'id': '0x10de'}}, + 'pci_0001_1b_00_0': {'bus': 27, + 'device_type': 'pci', + 'domain': 1, + 'driver': {'name': 'nvidia'}, + 'function': 0, + 'iommuGroup': 7, + 'name': 'pci_0001_1b_00_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0001:1b:00.0', + 'product': {'description': + 'GK210GL [Tesla K80]', + 'id': '0x0302'}, + 'slot': 0, + 'vga3d': True, + 'vendor': {'description': 'NVIDIA Corp', + 'id': '0x10de'}}, + 'pci_0000_1c_00_0': {'bus': 28, + 'device_type': 'pci', + 'domain': 0, + 'driver': {'name': 'nvidia'}, + 'function': 0, + 'iommuGroup': 7, + 'name': 'pci_0000_1c_00_0', + 'parent': 'computer', + 'path': + '/sys/devices/pci0000:00/0000:0d:00.0', + 'product': {'description': + 'GK210GL [Tesla K80]', + 'id': '0x0302'}, + 'slot': 0, + 'vga3d': True, + 'vendor': {'description': 'NVIDIA Corp', + 'id': '0x10de'}}, 'scsi_host0': {'adapter': {'fabric_wwn': '37df6c1efa1b4388', 'type': 'fc_host', 'wwnn': 'efb6563f06434a98', diff --git a/tests/test_mockmodel.py b/tests/test_mockmodel.py index 5c9fad4..0668191 100644 --- a/tests/test_mockmodel.py +++ b/tests/test_mockmodel.py @@ -26,6 +26,7 @@ import unittest from tests.utils import get_free_port, patch_auth, request, run_server from tests.utils import wait_task +from wok.exception import InvalidOperation from wok.plugins.kimchi import mockmodel from wok.plugins.kimchi.osinfo import get_template_default @@ -112,6 +113,38 @@ class MockModelTests(unittest.TestCase): vms.append(u'test') self.assertEqual(model.vms_get_list(), sorted(vms)) + def test_memory_window_changes(self): + model.templates_create({'name': u'test', + 'cdrom': fake_iso}) + task = model.vms_create({'name': u'test-vm', + 'template': '/plugins/kimchi/templates/test'}) + wait_task(model.task_lookup, task['id']) + + info = model.device_lookup('pci_0000_1a_00_0') + model.vmhostdevs_update_mmio_guest(u'test-vm', True) + model._attach_device(u'test-vm', + model._get_pci_device_xml(info, 0, False)) + + def test_hotplug_3D_card(self): + model.templates_create({'name': u'test', + 'cdrom': fake_iso}) + task = model.vms_create({'name': u'test-vm', + 'template': '/plugins/kimchi/templates/test'}) + wait_task(model.task_lookup, task['id']) + model.vm_start(u'test-vm') + + # attach the 3D cards found to a running guest + all_devices = model.devices_get_list() + for device in all_devices: + device_info = model.device_lookup(device) + if model.device_is_device_3D_controller(device_info): + try: + model.vmhostdevs_create(u'test-vm', {'name': device}) + + # expect the error: KCHVMHDEV0006E + except InvalidOperation as e: + self.assertEqual(e.message[:14], u'KCHVMHDEV0006E') + def test_vm_info(self): model.templates_create({'name': u'test', 'cdrom': fake_iso}) -- 1.9.1
Hi Ziviani, One test case is failing for me: ====================================================================== FAIL: test_hotplug_3D_card (test_mockmodel.MockModelTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "test_mockmodel.py", line 146, in test_hotplug_3D_card self.assertEqual(e.message[:14], u'KCHVMHDEV0006E') AssertionError: u'KCHVMHDEV0003E' != u'KCHVMHDEV0006E' - KCHVMHDEV0003E ? ^ + KCHVMHDEV0006E ? ^ ---------------------------------------------------------------------- Ran 120 tests in 994.219s Maybe it is because my host does not have IOMMU enabled. Could you do the adjustments needed in the test case and resend the patch? Thanks, Aline Manera On 02/26/2016 03:28 PM, Jose Ricardo Ziviani wrote:
Set the window size accordingly when attaching a GPU.
Jose Ricardo Ziviani (2): Set mmio memory when GPU is attached Create test cases for GPU attachment
i18n.py | 2 +- mockmodel.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++- model/host.py | 4 +- model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++-- tests/test_mockmodel.py | 33 ++++++++++ 5 files changed, 332 insertions(+), 8 deletions(-)
participants (2)
-
Aline Manera -
Jose Ricardo Ziviani