11:12 a.m.
v2:
- ignore iommu validation for testing gpu attachment.
Set the window size accordingly when attaching a GPU.
Jose Ricardo Ziviani (2):
Set mmio memory when GPU is attached
Create test cases for GPU attachment
i18n.py | 2 +-
mockmodel.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++-
model/host.py | 4 +-
model/vmhostdevs.py | 141 +++++++++++++++++++++++++++++++++++++++--
tests/test_mockmodel.py | 33 ++++++++++
5 files changed, 336 insertions(+), 8 deletions(-)
--
1.9.1
11:12 a.m.
New subject: [PATCH] [Kimchi v2 1/2] Set mmio memory when GPU is attached
Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com>
---
i18n.py | 2 +-
model/host.py | 4 +-
model/vmhostdevs.py | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++--
3 files changed, 140 insertions(+), 7 deletions(-)
diff --git a/i18n.py b/i18n.py
index a953b55..008e327 100644
--- a/i18n.py
+++ b/i18n.py
@@ -144,7 +144,7 @@ messages = {
"For AMD CPU, add 'iommu=pt iommu=1'."),
"KCHVMHDEV0004E": _('"name" should be a device name
string'),
"KCHVMHDEV0005E": _('The device %(name)s is probably in use by the
host. Unable to attach it to the guest.'),
- "KCHVMHDEV0006E": _('Hot-plug of device %(name)s is not
supported.'),
+ "KCHVMHDEV0006E": _('Hot-(un)plug of device %(name)s is not
supported.'),
"KCHVMIF0001E": _("Interface %(iface)s does not exist in virtual
machine %(name)s"),
"KCHVMIF0002E": _("Network %(network)s specified for virtual machine
%(name)s does not exist"),
diff --git a/model/host.py b/model/host.py
index 135c8a9..abf1191 100644
--- a/model/host.py
+++ b/model/host.py
@@ -146,9 +146,9 @@ class DevicesModel(object):
class DeviceModel(object):
def __init__(self, **kargs):
self.conn = kargs['conn']
- self.iommu_groups = self._get_iommu_groups()
+ self.iommu_groups = self.get_iommu_groups()
- def _get_iommu_groups(self):
+ def get_iommu_groups(self):
iommu_groups = defaultdict(list)
conn = self.conn
diff --git a/model/vmhostdevs.py b/model/vmhostdevs.py
index dabe8e4..2a62249 100644
--- a/model/vmhostdevs.py
+++ b/model/vmhostdevs.py
@@ -22,7 +22,7 @@ import libvirt
import os
import platform
from lxml import etree, objectify
-from lxml.builder import E
+from lxml.builder import E, ElementMaker
from operator import itemgetter
from wok.exception import InvalidOperation, InvalidParameter, NotFoundError
@@ -34,6 +34,12 @@ from wok.plugins.kimchi.model.config import CapabilitiesModel
from wok.plugins.kimchi.model.host import DeviceModel, DevicesModel
from wok.plugins.kimchi.model.utils import get_vm_config_flag
from wok.plugins.kimchi.model.vms import DOM_STATE_MAP, VMModel
+from wok.plugins.kimchi.xmlutils.qemucmdline import get_qemucmdline_xml
+from wok.plugins.kimchi.xmlutils.qemucmdline import QEMU_NAMESPACE
+
+
+CMDLINE_FIELD_NAME = 'spapr-pci-host-bridge.mem_win_size'
+WINDOW_SIZE_BAR = 0x800000000
class VMHostDevsModel(object):
@@ -149,6 +155,7 @@ class VMHostDevsModel(object):
slots = sorted(slots)
+ free = 0
for free, slot in enumerate(slots, start=1):
if free < slot:
return free
@@ -184,8 +191,9 @@ class VMHostDevsModel(object):
DOM_STATE_MAP[dom.info()[0]] == "shutoff"
pci_infos = sorted(pci_infos, key=itemgetter('name'))
- if dev_model.is_device_3D_controller(dev_info) and \
- DOM_STATE_MAP[dom.info()[0]] != "shutoff":
+ # does not allow hot-plug of 3D graphic cards
+ is_3D_device = dev_model.is_device_3D_controller(dev_info)
+ if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff":
raise InvalidOperation('KCHVMHDEV0006E',
{'name': dev_info['name']})
@@ -207,6 +215,12 @@ class VMHostDevsModel(object):
device_flags = get_vm_config_flag(dom, mode='all')
+ # when attaching a 3D graphic device it might be necessary to increase
+ # the window size memory in order to be able to attach more than one
+ # device to the same guest
+ if is_3D_device:
+ self.update_mmio_guest(vmid, True)
+
slot = 0
if is_multifunction:
slot = self._available_slot(dom)
@@ -229,6 +243,115 @@ class VMHostDevsModel(object):
return dev_info['name']
+ def _count_3D_devices_attached(self, dom):
+ counter = 0
+ root = objectify.fromstring(dom.XMLDesc(0))
+
+ try:
+ hostdev = root.devices.hostdev
+
+ except AttributeError:
+ return counter
+
+ for device in hostdev:
+ if device.attrib['type'] != 'pci':
+ continue
+
+ name = DeviceModel.deduce_dev_name(device, self.conn)
+ info = DeviceModel(conn=self.conn).lookup(name)
+ if 'vga3d' in info and info['vga3d']:
+ counter += 1
+
+ return counter
+
+ def update_mmio_guest(self, vmid, is_attaching):
+ dom = VMModel.get_vm(vmid, self.conn)
+ # get the number of 3D graphic cards already attached to the guest
+ # based on this number we will decide if the memory size will be
+ # increased or not
+ counter = self._count_3D_devices_attached(dom)
+ if counter == 0 and is_attaching:
+ return
+
+ size = 0
+ if is_attaching:
+ # suppose this is the 3rd graphic card to be attached to the same
+ # guest, counter will be 2+1 (2 existing + this attachment) times
+ # 32G (0x80000000)
+ size = hex((counter + 1) * WINDOW_SIZE_BAR)
+
+ else:
+ size = hex(counter * WINDOW_SIZE_BAR)
+
+ # if the guest already has the xml file we will simply update the
+ # value, otherwise we will add the new field
+ new_xml = self._update_win_memory_size(dom, counter, size)
+ if new_xml is None and is_attaching:
+ new_xml = self._add_win_memory_size(dom, size)
+
+ # update the XML
+ self.conn.get().defineXML(new_xml)
+
+ def _update_win_memory_size(self, dom, counter, wnd_size):
+ root = objectify.fromstring(dom.XMLDesc(0))
+
+ # look for the existing argument in <qemu:commandline> and try
+ # to update the value (or remove if there is only one (or none)
+ # graphic card attached.
+ cmdline = root.findall('{%s}commandline' % QEMU_NAMESPACE)
+ for line in cmdline:
+ for arg in line.iterchildren():
+ if not arg.values()[0].startswith(CMDLINE_FIELD_NAME):
+ continue
+
+ # update mem_win_size value
+ if counter > 1:
+ arg.set('value', CMDLINE_FIELD_NAME + '=' +
wnd_size)
+
+ # remove mem_win_size
+ elif counter <= 1:
+ line.remove(arg)
+
+ return etree.tostring(root, encoding='utf-8',
+ pretty_print=True)
+
+ return None
+
+ def _add_win_memory_size(self, dom, wnd_size):
+ root = objectify.fromstring(dom.XMLDesc(0))
+ val = CMDLINE_FIELD_NAME + '=' + wnd_size
+
+ cmdline = root.find('{%s}commandline' % QEMU_NAMESPACE)
+ # <qemu:commandline> doesn't exist, create the full commandline xml
+ # with the required values and return
+ if cmdline is None:
+ args = {}
+ args['-global'] = val
+ root.append(etree.fromstring(get_qemucmdline_xml(args)))
+ return etree.tostring(root, encoding='utf-8', pretty_print=True)
+
+ # <qemu:commandline> exists and already has the tag
+ # <qemu:arg value='-global'> (user could already been using this
for
+ # something else), so we just add our <qemu:arg...> missing.
+ found = False
+ for arg in cmdline.iterchildren():
+ if arg.values()[0] == '-global':
+ EM = ElementMaker(namespace=QEMU_NAMESPACE,
+ nsmap={'qemu': QEMU_NAMESPACE})
+ cmdline.append(EM.arg(value=val))
+ found = True
+ break
+
+ # <qemu:commandline> exists but there is no <qemu:arg value global>
+ # so, we add those missing arguments inside the exising cmdline
+ if not found:
+ EM = ElementMaker(namespace=QEMU_NAMESPACE,
+ nsmap={'qemu': QEMU_NAMESPACE})
+ cmdline.append(EM.arg(value='-global'))
+ cmdline.append(EM.arg(value=val))
+
+ return etree.tostring(root, encoding='utf-8', pretty_print=True)
+
def _get_scsi_device_xml(self, dev_info):
adapter = E.adapter(name=('scsi_host%s' % dev_info['host']))
address = E.address(type='scsi', bus=str(dev_info['bus']),
@@ -285,7 +408,8 @@ class VMHostDevModel(object):
'type': e.attrib['type'],
'product': dev_info.get('product', None),
'vendor': dev_info.get('vendor', None),
- 'multifunction': dev_info.get('multifunction',
None)}
+ 'multifunction': dev_info.get('multifunction',
None),
+ 'vga3d': dev_info.get('vga3d', None)}
raise NotFoundError('KCHVMHDEV0001E',
{'vmid': vmid, 'dev_name': dev_name})
@@ -304,6 +428,13 @@ class VMHostDevModel(object):
pci_devs = [(DeviceModel.deduce_dev_name(e, self.conn), e)
for e in hostdev if e.attrib['type'] == 'pci']
+ dev_model = DeviceModel(conn=self.conn)
+ dev_info = dev_model.lookup(dev_name)
+ is_3D_device = dev_model.is_device_3D_controller(dev_info)
+ if is_3D_device and DOM_STATE_MAP[dom.info()[0]] != "shutoff":
+ raise InvalidOperation('KCHVMHDEV0006E',
+ {'name': dev_info['name']})
+
for e in hostdev:
if DeviceModel.deduce_dev_name(e, self.conn) == dev_name:
xmlstr = etree.tostring(e)
@@ -311,6 +442,8 @@ class VMHostDevModel(object):
xmlstr, get_vm_config_flag(dom, mode='all'))
if e.attrib['type'] == 'pci':
self._delete_affected_pci_devices(dom, dev_name, pci_devs)
+ if is_3D_device:
+ self.update_mmio_guest(vmid, False)
break
else:
raise NotFoundError('KCHVMHDEV0001E',
--
1.9.1
11:12 a.m.
New subject: [PATCH] [Kimchi v2 2/2] Create test cases for GPU attachment
Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com>
---
mockmodel.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++-
tests/test_mockmodel.py | 33 ++++++++++
2 files changed, 196 insertions(+), 1 deletion(-)
diff --git a/mockmodel.py b/mockmodel.py
index d010df7..a765770 100644
--- a/mockmodel.py
+++ b/mockmodel.py
@@ -35,7 +35,9 @@ from wok.plugins.kimchi import imageinfo
from wok.plugins.kimchi import osinfo
from wok.plugins.kimchi.model import cpuinfo
from wok.plugins.kimchi.model import vmifaces
+from wok.plugins.kimchi.model.groups import PAMGroupsModel
from wok.plugins.kimchi.model.host import DeviceModel
+from wok.plugins.kimchi.model.host import DevicesModel
from wok.plugins.kimchi.model.libvirtstoragepool import IscsiPoolDef
from wok.plugins.kimchi.model.libvirtstoragepool import NetfsPoolDef
from wok.plugins.kimchi.model.libvirtstoragepool import StoragePoolDef
@@ -47,7 +49,7 @@ from wok.plugins.kimchi.model.storagevolumes import StorageVolumesModel
from wok.plugins.kimchi.model import storagevolumes
from wok.plugins.kimchi.model.templates import LibvirtVMTemplate
from wok.plugins.kimchi.model.users import PAMUsersModel
-from wok.plugins.kimchi.model.groups import PAMGroupsModel
+from wok.plugins.kimchi.model.vmhostdevs import VMHostDevsModel
from wok.plugins.kimchi.utils import pool_name_from_uri
from wok.plugins.kimchi.vmtemplate import VMTemplate
@@ -118,6 +120,10 @@ class MockModel(Model):
setattr(self, m, mock_method)
DeviceModel.lookup = self._mock_device_lookup
+ DeviceModel.get_iommu_groups = self._mock_device_get_iommu_groups
+ DeviceModel.is_device_3D_controller = \
+ self._mock_device_is_device_3D_controller
+ DevicesModel.get_list = self._mock_devices_get_list
StoragePoolsModel._check_lvm = self._check_lvm
StoragePoolModel._update_lvm_disks = self._update_lvm_disks
StorageVolumesModel.get_list = self._mock_storagevolumes_get_list
@@ -125,6 +131,8 @@ class MockModel(Model):
LibvirtVMTemplate._get_volume_path = self._get_volume_path
VMTemplate.get_iso_info = self._probe_image
imageinfo.probe_image = self._probe_image
+ VMHostDevsModel._get_pci_device_xml = self._get_pci_device_xml
+ VMHostDevsModel._validate_pci_passthrough_env = self._validate_pci
def reset(self):
MockModel._mock_vms = defaultdict(list)
@@ -326,6 +334,16 @@ class MockModel(Model):
return [dev['name'] for dev in self._mock_devices.devices.values()
if dev['device_type'] == _cap]
+ def _mock_device_get_iommu_groups(self):
+ return [dev['iommuGroup'] for dev in
+ self._mock_devices.devices.values()
+ if 'iommuGroup' in dev]
+
+ def _mock_device_is_device_3D_controller(self, info):
+ if 'driver' in info and 'name' in info['driver']:
+ return info['driver']['name'] == 'nvidia'
+ return False
+
def _mock_device_lookup(self, dev_name):
return self._mock_devices.devices[dev_name]
@@ -405,6 +423,46 @@ class MockModel(Model):
if sn.name == name:
sn.current = True
+ def _attach_device(self, vm_name, xmlstr):
+ MockModel._mock_vms[vm_name].append(xmlstr)
+
+ def _validate_pci(self):
+ pass
+
+ def _get_pci_device_xml(self, dev_info, slot, is_multifunction):
+ if 'detach_driver' not in dev_info:
+ dev_info['detach_driver'] = 'kvm'
+
+ source = E.source(E.address(domain=str(dev_info['domain']),
+ bus=str(dev_info['bus']),
+ slot=str(dev_info['slot']),
+ function=str(dev_info['function'])))
+ driver = E.driver(name=dev_info['detach_driver'])
+
+ if is_multifunction:
+ multi = E.address(type='pci',
+ domain='0',
+ bus='0',
+ slot=str(slot),
+ function=str(dev_info['function']))
+
+ if dev_info['function'] == 0:
+ multi = E.address(type='pci',
+ domain='0',
+ bus='0',
+ slot=str(slot),
+ function=str(dev_info['function']),
+ multifunction='on')
+
+ host_dev = E.hostdev(source, driver, multi,
+ mode='subsystem', type='pci',
managed='yes')
+
+ else:
+ host_dev = E.hostdev(source, driver,
+ mode='subsystem', type='pci',
managed='yes')
+
+ return ET.tostring(host_dev)
+
class MockStorageVolumes(object):
def __init__(self):
@@ -481,6 +539,7 @@ class MockDevices(object):
'Centrino Advanced-N 6205 [Taylor Peak]',
'id': '0x0085'},
'slot': 0,
+ 'vga3d': False,
'vendor': {'description': 'Intel
Corporation',
'id': '0x8086'}},
'pci_0000_0d_00_0': {'bus': 13,
@@ -497,8 +556,111 @@ class MockDevices(object):
'PCIe SDXC/MMC Host Controller',
'id': '0xe823'},
'slot': 0,
+ 'vga3d': False,
'vendor': {'description': 'Ricoh Co
Ltd',
'id': '0x1180'}},
+ 'pci_0000_09_01_0': {'bus': 9,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'tg3'},
+ 'function': 0,
+ 'iommuGroup': 8,
+ 'name': 'pci_0000_09_01_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:09:01.0',
+ 'product': {'description':
+ 'NetXtreme BCM5719',
+ 'id': '0x1657'},
+ 'slot': 1,
+ 'vga3d': False,
+ 'vendor': {'description': 'Broadcom
Corp',
+ 'id': '0x14e4'}},
+ 'pci_0000_09_01_1': {'bus': 9,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'tg3'},
+ 'function': 1,
+ 'iommuGroup': 8,
+ 'name': 'pci_0000_09_01_1',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:09:01.1',
+ 'product': {'description':
+ 'NetXtreme BCM5719',
+ 'id': '0x1657'},
+ 'slot': 1,
+ 'vga3d': False,
+ 'vendor': {'description': 'Broadcom
Corp',
+ 'id': '0x14e4'}},
+ 'pci_0000_09_01_2': {'bus': 9,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'tg3'},
+ 'function': 2,
+ 'iommuGroup': 8,
+ 'name': 'pci_0000_09_01_2',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:09:01.2',
+ 'product': {'description':
+ 'NetXtreme BCM5719',
+ 'id': '0x1657'},
+ 'slot': 1,
+ 'vga3d': False,
+ 'vendor': {'description': 'Broadcom
Corp',
+ 'id': '0x14e4'}},
+ 'pci_0000_1a_00_0': {'bus': 26,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'nvidia'},
+ 'function': 0,
+ 'iommuGroup': 9,
+ 'name': 'pci_0000_1a_00_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:1a:00.0',
+ 'product': {'description':
+ 'GK210GL [Tesla K80]',
+ 'id': '0x0302'},
+ 'slot': 0,
+ 'vga3d': True,
+ 'vendor': {'description': 'NVIDIA
Corp',
+ 'id': '0x10de'}},
+ 'pci_0001_1b_00_0': {'bus': 27,
+ 'device_type': 'pci',
+ 'domain': 1,
+ 'driver': {'name': 'nvidia'},
+ 'function': 0,
+ 'iommuGroup': 7,
+ 'name': 'pci_0001_1b_00_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0001:1b:00.0',
+ 'product': {'description':
+ 'GK210GL [Tesla K80]',
+ 'id': '0x0302'},
+ 'slot': 0,
+ 'vga3d': True,
+ 'vendor': {'description': 'NVIDIA
Corp',
+ 'id': '0x10de'}},
+ 'pci_0000_1c_00_0': {'bus': 28,
+ 'device_type': 'pci',
+ 'domain': 0,
+ 'driver': {'name': 'nvidia'},
+ 'function': 0,
+ 'iommuGroup': 7,
+ 'name': 'pci_0000_1c_00_0',
+ 'parent': 'computer',
+ 'path':
+ '/sys/devices/pci0000:00/0000:0d:00.0',
+ 'product': {'description':
+ 'GK210GL [Tesla K80]',
+ 'id': '0x0302'},
+ 'slot': 0,
+ 'vga3d': True,
+ 'vendor': {'description': 'NVIDIA
Corp',
+ 'id': '0x10de'}},
'scsi_host0': {'adapter': {'fabric_wwn':
'37df6c1efa1b4388',
'type': 'fc_host',
'wwnn': 'efb6563f06434a98',
diff --git a/tests/test_mockmodel.py b/tests/test_mockmodel.py
index 5c9fad4..0668191 100644
--- a/tests/test_mockmodel.py
+++ b/tests/test_mockmodel.py
@@ -26,6 +26,7 @@ import unittest
from tests.utils import get_free_port, patch_auth, request, run_server
from tests.utils import wait_task
+from wok.exception import InvalidOperation
from wok.plugins.kimchi import mockmodel
from wok.plugins.kimchi.osinfo import get_template_default
@@ -112,6 +113,38 @@ class MockModelTests(unittest.TestCase):
vms.append(u'test')
self.assertEqual(model.vms_get_list(), sorted(vms))
+ def test_memory_window_changes(self):
+ model.templates_create({'name': u'test',
+ 'cdrom': fake_iso})
+ task = model.vms_create({'name': u'test-vm',
+ 'template':
'/plugins/kimchi/templates/test'})
+ wait_task(model.task_lookup, task['id'])
+
+ info = model.device_lookup('pci_0000_1a_00_0')
+ model.vmhostdevs_update_mmio_guest(u'test-vm', True)
+ model._attach_device(u'test-vm',
+ model._get_pci_device_xml(info, 0, False))
+
+ def test_hotplug_3D_card(self):
+ model.templates_create({'name': u'test',
+ 'cdrom': fake_iso})
+ task = model.vms_create({'name': u'test-vm',
+ 'template':
'/plugins/kimchi/templates/test'})
+ wait_task(model.task_lookup, task['id'])
+ model.vm_start(u'test-vm')
+
+ # attach the 3D cards found to a running guest
+ all_devices = model.devices_get_list()
+ for device in all_devices:
+ device_info = model.device_lookup(device)
+ if model.device_is_device_3D_controller(device_info):
+ try:
+ model.vmhostdevs_create(u'test-vm', {'name':
device})
+
+ # expect the error: KCHVMHDEV0006E
+ except InvalidOperation as e:
+ self.assertEqual(e.message[:14], u'KCHVMHDEV0006E')
+
def test_vm_info(self):
model.templates_create({'name': u'test',
'cdrom': fake_iso})
--
1.9.1
1:54 p.m.
Reviewed-By: Ramon Medeiros <ramonn(a)br.ibm.com>
On 03/01/2016 02:12 PM, Jose Ricardo Ziviani wrote:
v2:
- ignore iommu validation for testing gpu attachment.
Set the window size accordingly when attaching a GPU.
Jose Ricardo Ziviani (2):
Set mmio memory when GPU is attached
Create test cases for GPU attachment
i18n.py | 2 +-
mockmodel.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++-
model/host.py | 4 +-
model/vmhostdevs.py | 141 +++++++++++++++++++++++++++++++++++++++--
tests/test_mockmodel.py | 33 ++++++++++
5 files changed, 336 insertions(+), 8 deletions(-)
--
Ramon Nunes Medeiros
Kimchi Developer
Linux Technology Center Brazil
IBM Systems & Technology Group
Phone : +55 19 2132 7878
ramonn(a)br.ibm.com
3192
days inactive
3193
days old
4 comments
3 participants
participants (3)
-
Aline Manera -
Jose Ricardo Ziviani -
Ramon Medeiros