12:28 p.m.
Set the window size accordingly when attaching a GPU.
Jose Ricardo Ziviani (2):
Set mmio memory when GPU is attached
Create test cases for GPU attachment
i18n.py | 2 +-
mockmodel.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++-
model/host.py | 4 +-
model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++--
tests/test_mockmodel.py | 33 ++++++++++
5 files changed, 332 insertions(+), 8 deletions(-)
--
1.9.1
12:28 p.m.
New subject: [PATCH] [Kimchi 1/2] Set mmio memory when GPU is attached
Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com>
---
i18n.py | 2 +-
model/host.py | 4 +-
model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++--
3 files changed, 140 insertions(+), 7 deletions(-)
diff --git a/i18n.py b/i18n.py
index a953b55..008e327 100644
--- a/i18n.py
+++ b/i18n.py
@@ -144,7 +144,7 @@ messages = {
"For AMD CPU, add 'iommu=pt iommu=1'."),
"KCHVMHDEV0004E": _('"name" should be a device name
string'),
"KCHVMHDEV0005E": _('The device %(name)s is probably in use by the
host. Unable to attach it to the guest.'),
- "KCHVMHDEV0006E": _('Hot-plug of device %(name)s is not
supported.'),
+ "KCHVMHDEV0006E": _('Hot-(un)plug of device %(name)s is not
supported.'),
"KCHVMIF0001E": _("Interface %(iface)s does not exist in virtual
machine %(name)s"),
"KCHVMIF0002E": _("Network %(network)s specified for virtual machine
%(name)s does not exist"),
diff --git a/model/host.py b/model/host.py
index 135c8a9..abf1191 100644
--- a/model/host.py
+++ b/model/host.py
@@ -146,9 +146,9 @@ class DevicesModel(object):
class DeviceModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
- self.iommu_groups = self._get_iommu_groups()
+ self.iommu_groups = self.get_iommu_groups()
- def _get_iommu_groups(self):
+ def get_iommu_groups(self):
iommu_groups = defaultdict(list)
conn = self.conn
diff --git a/model/vmhostdevs.py b/model/vmhostdevs.py
index dabe8e4..2a62249 100644
--- a/model/vmhostdevs.py
+++ b/model/vmhostdevs.py
@@ -22,7 +22,7 @@ import libvirt
import os
import platform
from lxml import etree, objectify
-from lxml.builder import E
+from lxml.builder import E, ElementMaker
from operator import itemgetter
from wok.exception import InvalidOperation, InvalidParameter, NotFoundError
@@ -34,6 +34,12 @@ from wok.plugins.kimchi.model.config import CapabilitiesModel
from wok.plugins.kimchi.model.host import DeviceModel, DevicesModel
from wok.plugins.kimchi.model.utils import get_vm_config_flag
from wok.plugins.kimchi.model.vms import DOM_STATE_MAP, VMModel
+from wok.plugins.kimchi.xmlutils.qemucmdline import get_qemucmdline_xml
+from wok.plugins.kimchi.xmlutils.qemucmdline import QEMU_NAMESPACE
+
+
+CMDLINE_FIELD_NAME = 'spapr-pci-host-bridge.mem_win_size'
+WINDOW_SIZE_BAR = 0x800000000
class VMHostDevsModel(object):
@@ -149,6 +155,7 @@ class VMHostDevsModel(object):
slots = sorted(slots)
+ free = 0
for free, slot in enumerate(slots, start=1):
if free < slot:
return free
@@ -184,8 +191,9 @@ class VMHostDevsModel(object):
DOM_STATE_MAP[dom.info()[0]] == "shutoff"
pci_infos = sorted(pci_infos, key=itemgetter('name'))
- if dev_model.is_device_3D_controller(dev_info) and \
- DOM_STATE_MAP[dom.info()[0]] != "shutoff":
+ # does not allow hot-plug of 3D graphic cards
+ is_3D_device = dev_model.is_device_3D_controller(dev_info)
+ if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff":
raise InvalidOperation('KCHVMHDEV0006E',
{'name': dev_info['name']})
@@ -207,6 +215,12 @@ class VMHostDevsModel(object):
device_flags = get_vm_config_flag(dom, mode='all')
+ # when attaching a 3D graphic device it might be necessary to increase
+ # the window size memory in order to be able to attach more than one
+ # device to the same guest
+ if is_3D_device:
+ self.update_mmio_guest(vmid, True)
+
slot = 0
if is_multifunction:
slot = self._available_slot(dom)
@@ -229,6 +243,115 @@ class VMHostDevsModel(object):
return dev_info['name']
+ def _count_3D_devices_attached(self, dom):
+ counter = 0
+ root = objectify.fromstring(dom.XMLDesc(0))
+
+ try:
+ hostdev = root.devices.hostdev
+
+ except AttributeError:
+ return counter
+
+ for device in hostdev:
+ if device.attrib['type'] != 'pci':
+ continue
+
+ name = DeviceModel.deduce_dev_name(device, self.conn)
+ info = DeviceModel(conn=self.conn).lookup(name)
+ if 'vga3d' in info and info['vga3d']:
+ counter += 1
+
+ return counter
+
+ def update_mmio_guest(self, vmid, is_attaching):
+ dom = VMModel.get_vm(vmid, self.conn)
+ # get the number of 3D graphic cards already attached to the guest
+ # based on this number we will decide if the memory size will be
+ # increased or not
+ counter = self._count_3D_devices_attached(dom)
+ if counter == 0 and is_attaching:
+ return
+
+ size = 0
+ if is_attaching:
+ # suppose this is the 3rd graphic card to be attached to the same
+ # guest, counter will be 2+1 (2 existing + this attachment) times
+ # 32G (0x80000000)
+ size = hex((counter + 1) * WINDOW_SIZE_BAR)
+
+ else:
+ size = hex(counter * WINDOW_SIZE_BAR)
+
+ # if the guest already has the xml file we will simply update the
+ # value, otherwise we will add the new field
+ new_xml = self._update_win_memory_size(dom, counter, size)
+ if new_xml is None and is_attaching:
+ new_xml = self._add_win_memory_size(dom, size)
+
+ # update the XML
+ self.conn.get().defineXML(new_xml)
+
+ def _update_win_memory_size(self, dom, counter, wnd_size):
+ root = objectify.fromstring(dom.XMLDesc(0))
+
+ # look for the existing argument in <qemu:commandline> and try
+ # to update the value (or remove if there is only one (or none)
+ # graphic card attached.
+ cmdline = root.findall('{%s}commandline' % QEMU_NAMESPACE)
+ for line in cmdline:
+ for arg in line.iterchildren():
+ if not arg.values()[0].startswith(CMDLINE_FIELD_NAME):
+ continue
+
+ # update mem_win_size value
+ if counter > 1:
+ arg.set('value', CMDLINE_FIELD_NAME + '=' +
wnd_size)
+
+ # remove mem_win_size
+ elif counter <= 1:
+ line.remove(arg)
+
+ return etree.tostring(root, encoding='utf-8',
+ pretty_print=True)
+
+ return None
+
+ def _add_win_memory_size(self, dom, wnd_size):
+ root = objectify.fromstring(dom.XMLDesc(0))
+ val = CMDLINE_FIELD_NAME + '=' + wnd_size
+
+ cmdline = root.find('{%s}commandline' % QEMU_NAMESPACE)
+ # <qemu:commandline> doesn't exist, create the full commandline xml
+ # with the required values and return
+ if cmdline is None:
+ args = {}
+ args['-global'] = val
+ root.append(etree.fromstring(get_qemucmdline_xml(args)))
+ return etree.tostring(root, encoding='utf-8', pretty_print=True)
+
+ # <qemu:commandline> exists and already has the tag
+ # <qemu:arg value='-global'> (user could already been using this
for
+ # something else), so we just add our <qemu:arg...> missing.
+ found = False
+ for arg in cmdline.iterchildren():
+ if arg.values()[0] == '-global':
+ EM = ElementMaker(namespace=QEMU_NAMESPACE,
+ nsmap={'qemu': QEMU_NAMESPACE})
+ cmdline.append(EM.arg(value=val))
+ found = True
+ break
+
+ # <qemu:commandline> exists but there is no <qemu:arg value global>
+ # so, we add those missing arguments inside the exising cmdline
+ if not found:
+ EM = ElementMaker(namespace=QEMU_NAMESPACE,
+ nsmap={'qemu': QEMU_NAMESPACE})
+ cmdline.append(EM.arg(value='-global'))
+ cmdline.append(EM.arg(value=val))
+
+ return etree.tostring(root, encoding='utf-8', pretty_print=True)
+
def _get_scsi_device_xml(self, dev_info):
adapter = E.adapter(name=('scsi_host%s' % dev_info['host']))
address = E.address(type='scsi', bus=str(dev_info['bus']),
@@ -285,7 +408,8 @@ class VMHostDevModel(object):
'type': e.attrib['type'],
'product': dev_info.get('product', None),
'vendor': dev_info.get('vendor', None),
- 'multifunction': dev_info.get('multifunction',
None)}
+ 'multifunction': dev_info.get('multifunction',
None),
+ 'vga3d': dev_info.get('vga3d', None)}
raise NotFoundError('KCHVMHDEV0001E',
{'vmid': vmid, 'dev_name': dev_name})
@@ -304,6 +428,13 @@ class VMHostDevModel(object):
pci_devs = [(DeviceModel.deduce_dev_name(e, self.conn), e)
for e in hostdev if e.attrib['type'] == 'pci']
+ dev_model = DeviceModel(conn=self.conn)
+ dev_info = dev_model.lookup(dev_name)
+ is_3D_device = dev_model.is_device_3D_controller(dev_info)
+ if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff":
+ raise InvalidOperation('KCHVMHDEV0006E',
+ {'name': dev_info['name']})
+
for e in hostdev:
if DeviceModel.deduce_dev_name(e, self.conn) == dev_name:
xmlstr = etree.tostring(e)
@@ -311,6 +442,8 @@ class VMHostDevModel(object):
xmlstr, get_vm_config_flag(dom, mode='all'))
if e.attrib['type'] == 'pci':
self._delete_affected_pci_devices(dom, dev_name, pci_devs)
+ if is_3D_device:
+ self.update_mmio_guest(vmid, False)
break
else:
raise NotFoundError('KCHVMHDEV0001E',
--
1.9.1
12:28 p.m.
New subject: [PATCH] [Kimchi 2/2] Create test cases for GPU attachment
Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com>
---
mockmodel.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++-
tests/test_mockmodel.py | 33 ++++++++++
2 files changed, 192 insertions(+), 1 deletion(-)
diff --git a/mockmodel.py b/mockmodel.py
index d010df7..034e6f6 100644
--- a/mockmodel.py
+++ b/mockmodel.py
@@ -35,7 +35,9 @@ from wok.plugins.kimchi import imageinfo
from wok.plugins.kimchi import osinfo
from wok.plugins.kimchi.model import cpuinfo
from wok.plugins.kimchi.model import vmifaces
+from wok.plugins.kimchi.model.groups import PAMGroupsModel
from wok.plugins.kimchi.model.host import DeviceModel
+from wok.plugins.kimchi.model.host import DevicesModel
from wok.plugins.kimchi.model.libvirtstoragepool import IscsiPoolDef
from wok.plugins.kimchi.model.libvirtstoragepool import NetfsPoolDef
from wok.plugins.kimchi.model.libvirtstoragepool import StoragePoolDef
@@ -47,7 +49,7 @@ from wok.plugins.kimchi.model.storagevolumes import StorageVolumesModel
from wok.plugins.kimchi.model import storagevolumes
from wok.plugins.kimchi.model.templates import LibvirtVMTemplate
from wok.plugins.kimchi.model.users import PAMUsersModel
-from wok.plugins.kimchi.model.groups import PAMGroupsModel
+from wok.plugins.kimchi.model.vmhostdevs import VMHostDevsModel
from wok.plugins.kimchi.utils import pool_name_from_uri
from wok.plugins.kimchi.vmtemplate import VMTemplate
@@ -118,6 +120,10 @@ class MockModel(Model):
setattr(self, m, mock_method)
DeviceModel.lookup = self._mock_device_lookup
+ DeviceModel.get_iommu_groups = self._mock_device_get_iommu_groups
+ DeviceModel.is_device_3D_controller = \
+ self._mock_device_is_device_3D_controller
+ DevicesModel.get_list = self._mock_devices_get_list
StoragePoolsModel._check_lvm = self._check_lvm
StoragePoolModel._update_lvm_disks = self._update_lvm_disks
StorageVolumesModel.get_list = self._mock_storagevolumes_get_list
@@ -125,6 +131,7 @@ class MockModel(Model):
LibvirtVMTemplate._get_volume_path = self._get_volume_path
VMTemplate.get_iso_info = self._probe_image
imageinfo.probe_image = self._probe_image
+ VMHostDevsModel._get_pci_device_xml = self._get_pci_device_xml
def reset(self):
MockModel._mock_vms = defaultdict(list)
@@ -326,6 +333,16 @@ class MockModel(Model):
return [dev['name'] for dev in self._mock_devices.devices.values()
if dev['device_type'] == _cap]
+ def _mock_device_get_iommu_groups(self):
+ return [dev['iommuGroup'] for dev in
+ self._mock_devices.devices.values()
+ if 'iommuGroup' in dev]
+
+ def _mock_device_is_device_3D_controller(self, info):
+ if 'driver' in info and 'name' in info['driver']:
+ return info['driver']['name'] == 'nvidia'
+ return False
+
def _mock_device_lookup(self, dev_name):
return self._mock_devices.devices[dev_name]
@@ -405,6 +422,43 @@ class MockModel(Model):
if sn.name == name:
sn.current = True
+ def _attach_device(self, vm_name, xmlstr):
+ MockModel._mock_vms[vm_name].append(xmlstr)
+
+ def _get_pci_device_xml(self, dev_info, slot, is_multifunction):
+ if 'detach_driver' not in dev_info:
+ dev_info['detach_driver'] = 'kvm'
+
+ source = E.source(E.address(domain=str(dev_info['domain']),
+ bus=str(dev_info['bus']),
+ slot=str(dev_info['slot']),
+ function=str(dev_info['function'])))
+ driver = E.driver(name=dev_info['detach_driver'])
+
+ if is_multifunction:
+ multi = E.address(type='pci',
+ domain='0',
+ bus='0',
+ slot=str(slot),
+ function=str(dev_info['function']))
+
+ if dev_info['function'] == 0:
+ multi = E.address(type='pci',
+ domain='0',
+ bus='0',
+ slot=str(slot),
+ function=str(dev_info['function']),
+ multifunction='on')
+
+ host_dev = E.hostdev(source, driver, multi,
+ mode='subsystem', type='pci',
managed='yes')
+
+ else:
+ host_dev = E.hostdev(source, driver,
+ mode='subsystem', type='pci',
managed='yes')
+
+ return ET.tostring(host_dev)
+
class MockStorageVolumes(object):
def __init__(self):
@@ -481,6 +535,7 @@ class MockDevices(object):
'Centrino Advanced-N 6205 [Taylor Peak]',
'id': '0x0085'},
'slot': 0,
+ 'vga3d': False,
'vendor': {'description': 'Intel
Corporation',
'id': '0x8086'}},
'pci_0000_0d_00_0': {'bus': 13,
@@ -497,8 +552,111 @@ class MockDevices(object):
'PCIe SDXC/MMC Host Controller',
'id': '0xe823'},
'slot': 0,
+ 'vga3d': False,
'vendor': {'description': 'Ricoh Co
Ltd',
'id': '0x1180'}},
+ 'pci_0000_09_01_0': {'bus': 9,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'tg3'},
+ 'function': 0,
+ 'iommuGroup': 8,
+ 'name': 'pci_0000_09_01_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:09:01.0',
+ 'product': {'description':
+ 'NetXtreme BCM5719',
+ 'id': '0x1657'},
+ 'slot': 1,
+ 'vga3d': False,
+ 'vendor': {'description': 'Broadcom
Corp',
+ 'id': '0x14e4'}},
+ 'pci_0000_09_01_1': {'bus': 9,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'tg3'},
+ 'function': 1,
+ 'iommuGroup': 8,
+ 'name': 'pci_0000_09_01_1',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:09:01.1',
+ 'product': {'description':
+ 'NetXtreme BCM5719',
+ 'id': '0x1657'},
+ 'slot': 1,
+ 'vga3d': False,
+ 'vendor': {'description': 'Broadcom
Corp',
+ 'id': '0x14e4'}},
+ 'pci_0000_09_01_2': {'bus': 9,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'tg3'},
+ 'function': 2,
+ 'iommuGroup': 8,
+ 'name': 'pci_0000_09_01_2',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:09:01.2',
+ 'product': {'description':
+ 'NetXtreme BCM5719',
+ 'id': '0x1657'},
+ 'slot': 1,
+ 'vga3d': False,
+ 'vendor': {'description': 'Broadcom
Corp',
+ 'id': '0x14e4'}},
+ 'pci_0000_1a_00_0': {'bus': 26,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'nvidia'},
+ 'function': 0,
+ 'iommuGroup': 9,
+ 'name': 'pci_0000_1a_00_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:1a:00.0',
+ 'product': {'description':
+ 'GK210GL [Tesla K80]',
+ 'id': '0x0302'},
+ 'slot': 0,
+ 'vga3d': True,
+ 'vendor': {'description': 'NVIDIA
Corp',
+ 'id': '0x10de'}},
+ 'pci_0001_1b_00_0': {'bus': 27,
+ 'device_type': 'pci',
+ 'domain': 1,
+ 'driver': {'name': 'nvidia'},
+ 'function': 0,
+ 'iommuGroup': 7,
+ 'name': 'pci_0001_1b_00_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0001:1b:00.0',
+ 'product': {'description':
+ 'GK210GL [Tesla K80]',
+ 'id': '0x0302'},
+ 'slot': 0,
+ 'vga3d': True,
+ 'vendor': {'description': 'NVIDIA
Corp',
+ 'id': '0x10de'}},
+ 'pci_0000_1c_00_0': {'bus': 28,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'nvidia'},
+ 'function': 0,
+ 'iommuGroup': 7,
+ 'name': 'pci_0000_1c_00_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:0d:00.0',
+ 'product': {'description':
+ 'GK210GL [Tesla K80]',
+ 'id': '0x0302'},
+ 'slot': 0,
+ 'vga3d': True,
+ 'vendor': {'description': 'NVIDIA
Corp',
+ 'id': '0x10de'}},
'scsi_host0': {'adapter': {'fabric_wwn':
'37df6c1efa1b4388',
'type': 'fc_host',
'wwnn': 'efb6563f06434a98',
diff --git a/tests/test_mockmodel.py b/tests/test_mockmodel.py
index 5c9fad4..0668191 100644
--- a/tests/test_mockmodel.py
+++ b/tests/test_mockmodel.py
@@ -26,6 +26,7 @@ import unittest
from tests.utils import get_free_port, patch_auth, request, run_server
from tests.utils import wait_task
+from wok.exception import InvalidOperation
from wok.plugins.kimchi import mockmodel
from wok.plugins.kimchi.osinfo import get_template_default
@@ -112,6 +113,38 @@ class MockModelTests(unittest.TestCase):
vms.append(u'test')
self.assertEqual(model.vms_get_list(), sorted(vms))
+ def test_memory_window_changes(self):
+ model.templates_create({'name': u'test',
+ 'cdrom': fake_iso})
+ task = model.vms_create({'name': u'test-vm',
+ 'template':
'/plugins/kimchi/templates/test'})
+ wait_task(model.task_lookup, task['id'])
+
+ info = model.device_lookup('pci_0000_1a_00_0')
+ model.vmhostdevs_update_mmio_guest(u'test-vm', True)
+ model._attach_device(u'test-vm',
+ model._get_pci_device_xml(info, 0, False))
+
+ def test_hotplug_3D_card(self):
+ model.templates_create({'name': u'test',
+ 'cdrom': fake_iso})
+ task = model.vms_create({'name': u'test-vm',
+ 'template':
'/plugins/kimchi/templates/test'})
+ wait_task(model.task_lookup, task['id'])
+ model.vm_start(u'test-vm')
+
+ # attach the 3D cards found to a running guest
+ all_devices = model.devices_get_list()
+ for device in all_devices:
+ device_info = model.device_lookup(device)
+ if model.device_is_device_3D_controller(device_info):
+ try:
+ model.vmhostdevs_create(u'test-vm', {'name':
device})
+
+ # expect the error: KCHVMHDEV0006E
+ except InvalidOperation as e:
+ self.assertEqual(e.message[:14], u'KCHVMHDEV0006E')
+
def test_vm_info(self):
model.templates_create({'name': u'test',
'cdrom': fake_iso})
--
1.9.1
1:56 p.m.
Hi Ziviani,
One test case is failing for me:
======================================================================
FAIL: test_hotplug_3D_card (test_mockmodel.MockModelTests)
----------------------------------------------------------------------
Traceback (most recent call last):
File "test_mockmodel.py", line 146, in test_hotplug_3D_card
self.assertEqual(e.message[:14], u'KCHVMHDEV0006E')
AssertionError: u'KCHVMHDEV0003E' != u'KCHVMHDEV0006E'
- KCHVMHDEV0003E
? ^
+ KCHVMHDEV0006E
? ^
----------------------------------------------------------------------
Ran 120 tests in 994.219s
Maybe it is because my host does not have IOMMU enabled.
Could you do the adjustments needed in the test case and resend the patch?
Thanks,
Aline Manera
On 02/26/2016 03:28 PM, Jose Ricardo Ziviani wrote:
Set the window size accordingly when attaching a GPU.
Jose Ricardo Ziviani (2):
Set mmio memory when GPU is attached
Create test cases for GPU attachment
i18n.py | 2 +-
mockmodel.py | 160 +++++++++++++++++++++++++++++++++++++++++++++++-
model/host.py | 4 +-
model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++--
tests/test_mockmodel.py | 33 ++++++++++
5 files changed, 332 insertions(+), 8 deletions(-)
3193
days inactive
3196
days old
3 comments
2 participants
participants (2)
-
Aline Manera -
Jose Ricardo Ziviani