[PATCHv5 0/6] Add volume reference count
From: Royce Lv <lvroyce@linux.vnet.ibm.com> tested: 1. sudo make check 2. from UI: list storage volumes and unrefered volume ref_cnt is 0, ref_cnt for refered ones are 1. Available volumes will be tracked by volume reference count. Add this field to storage volume, so that storage volume/pool action validation can rely on it. Royce Lv (6): Fix vm disk path when it does not have source element Add volume ref_cnt: update api.md Add volume ref_cnt: Update controller and json schema Add volume ref_cnt: Add model and mockmodel implementation Add volume ref_cnt: Update test Multiple pep8 fixes docs/API.md | 3 +++ src/kimchi/API.json | 25 +++++++++++++++++++++++++ src/kimchi/control/storagevolumes.py | 1 + src/kimchi/i18n.py | 4 ++++ src/kimchi/isoinfo.py | 4 ++-- src/kimchi/mockmodel.py | 10 +++++++--- src/kimchi/model/debugreports.py | 2 +- src/kimchi/model/storageservers.py | 4 ++-- src/kimchi/model/storagevolumes.py | 32 ++++++++++++++++++++++++++++++++ src/kimchi/model/templates.py | 2 +- src/kimchi/model/vmstorages.py | 23 +++++++++++++---------- tests/test_model.py | 10 +++++++--- tests/test_rest.py | 2 ++ 13 files changed, 100 insertions(+), 22 deletions(-) -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> When vm disk does not have source element, objectify library will error with 'no such element: source' Trap this exception. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/model/vmstorages.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/kimchi/model/vmstorages.py b/src/kimchi/model/vmstorages.py index 6298a81..c1c90ce 100644 --- a/src/kimchi/model/vmstorages.py +++ b/src/kimchi/model/vmstorages.py @@ -163,17 +163,20 @@ class VMStorageModel(object): if disk is None: raise NotFoundError("KCHCDROM0007E", {'dev_name': dev_name, 'vm_name': vm_name}) - source = disk.source path = "" - if source is not None: - src_type = disk.attrib['type'] - if src_type == 'network': - host = source.host - path = (source.attrib['protocol'] + '://' + - host.attrib['name'] + ':' + - host.attrib['port'] + source.attrib['name']) - else: - path = source.attrib[DEV_TYPE_SRC_ATTR_MAP[src_type]] + try: + source = disk.source + if source is not None: + src_type = disk.attrib['type'] + if src_type == 'network': + host = source.host + path = (source.attrib['protocol'] + '://' + + host.attrib['name'] + ':' + + host.attrib['port'] + source.attrib['name']) + else: + path = source.attrib[DEV_TYPE_SRC_ATTR_MAP[src_type]] + except: + pass dev_type = disk.attrib['device'] return {'dev': dev_name, 'type': dev_type, -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Update doc to support storage volume reference count. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- docs/API.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/API.md b/docs/API.md index fc39c65..d2b421a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -394,6 +394,9 @@ A interface represents available network interface on VM. * os_distro *(optional)*: os distribution of the volume, for iso volume only. * os_version *(optional)*: os version of the volume, for iso volume only. * bootable *(optional)*: True if iso image is bootable and not corrupted. + * ref_cnt: Number of vms which used this volume, + 0 for volumes which are available for attachment. + >1 indicate number of vms used this volume. * **DELETE**: Remove the Storage Volume * **POST**: *See Storage Volume Actions* -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Add ref_cnt to controller to report reference count information. Update json schema validation. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/API.json | 25 +++++++++++++++++++++++++ src/kimchi/control/storagevolumes.py | 1 + src/kimchi/i18n.py | 4 ++++ 3 files changed, 30 insertions(+) diff --git a/src/kimchi/API.json b/src/kimchi/API.json index f595bbf..b519fec 100644 --- a/src/kimchi/API.json +++ b/src/kimchi/API.json @@ -143,6 +143,31 @@ } } }, + "storagevolumes_create": { + "type": "object", + "error": "KCHVOL0015E", + "properties": { + "name": { + "description": "The name of the Storage Volume", + "type": "string", + "minLength": 1, + "required": true, + "error": "KCHVOL0012E" + }, + "allocation": { + "description": "The size(MiB) of allocation when create the storage volume", + "type": "number", + "minimum": 1, + "error": "KCHVOL0013E" + }, + "format": { + "description": "The format of the volume", + "type": "string", + "pattern": "^qcow2|raw$", + "error": "KCHVOL0014E" + } + } + }, "vms_create": { "type": "object", "error": "KCHVM0016E", diff --git a/src/kimchi/control/storagevolumes.py b/src/kimchi/control/storagevolumes.py index 718c97a..c4d6c41 100644 --- a/src/kimchi/control/storagevolumes.py +++ b/src/kimchi/control/storagevolumes.py @@ -49,6 +49,7 @@ class StorageVolume(Resource): 'capacity': self.info['capacity'], 'allocation': self.info['allocation'], 'path': self.info['path'], + 'ref_cnt': self.info['ref_cnt'], 'format': self.info['format']} for key in ('os_version', 'os_distro', 'bootable'): diff --git a/src/kimchi/i18n.py b/src/kimchi/i18n.py index f3e1803..a405d4b 100644 --- a/src/kimchi/i18n.py +++ b/src/kimchi/i18n.py @@ -151,6 +151,10 @@ messages = { "KCHVOL0009E": _("Unable to wipe storage volumes %(name)s. Details: %(err)s"), "KCHVOL0010E": _("Unable to delete storage volume %(name)s. Details: %(err)s"), "KCHVOL0011E": _("Unable to resize storage volume %(name)s. Details: %(err)s"), + "KCHVOL0012E": _("Storage volume name must be a string"), + "KCHVOL0013E": _("Storage volume allocation must be an integer number"), + "KCHVOL0014E": _("Storage volume format not supported"), + "KCHVOL0015E": _("Storage volume requires a volume name"), "KCHIFACE0001E": _("Interface %(name)s does not exist"), -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Set ref_cnt to 0 when new volume created, if the ref_cnt cannot be found, which means it is created outside of kimchi scope, report it when lookup storage volume, search info about storage volumes used by vms and count the ref_cnt. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/mockmodel.py | 3 +++ src/kimchi/model/storagevolumes.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py index f99b4cd..d69a58a 100644 --- a/src/kimchi/mockmodel.py +++ b/src/kimchi/mockmodel.py @@ -426,6 +426,7 @@ class MockModel(object): name = params['name'] volume = MockStorageVolume(pool, name, params) volume.info['type'] = params['type'] + volume.info['ref_cnt'] = params.get('ref_cnt', 0) volume.info['format'] = params['format'] volume.info['path'] = os.path.join( pool.info['path'], name) @@ -887,6 +888,7 @@ class MockVMTemplate(VMTemplate): disk_paths = [] for vol_info in volumes: vol_info['capacity'] = vol_info['capacity'] << 10 + vol_info['ref_cnt'] = 1 self.model.storagevolumes_create(pool.name, vol_info) disk_paths.append({'pool': pool.name, 'volume': vol_info['name']}) return disk_paths @@ -1005,6 +1007,7 @@ class MockStorageVolume(object): 'capacity': capacity << 20, 'allocation': params.get('allocation', '512'), 'path': params.get('path'), + 'ref_cnt': params.get('ref_cnt'), 'format': fmt} if fmt == 'iso': self.info['allocation'] = self.info['capacity'] diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index a031eb0..e9ce736 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -27,6 +27,8 @@ from kimchi.exception import MissingParameter, NotFoundError, OperationFailed from kimchi.isoinfo import IsoImage from kimchi.model.storagepools import StoragePoolModel from kimchi.utils import kimchi_log +from kimchi.model.vms import VMsModel +from kimchi.model.vmstorages import VMStoragesModel, VMStorageModel VOLUME_TYPE_MAP = {0: 'file', @@ -38,6 +40,7 @@ VOLUME_TYPE_MAP = {0: 'file', class StorageVolumesModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] + self.objstore = kargs['objstore'] def create(self, pool, params): vol_xml = """ @@ -56,6 +59,7 @@ class StorageVolumesModel(object): params.setdefault('format', 'qcow2') name = params['name'] + vol_id = '%s:%s' % (pool, name) try: pool = StoragePoolModel.get_storagepool(pool, self.conn) xml = vol_xml % params @@ -69,6 +73,10 @@ class StorageVolumesModel(object): raise OperationFailed("KCHVOL0007E", {'name': name, 'pool': pool, 'err': e.get_error_message()}) + + with self.objstore as session: + session.store('storagevolume', vol_id, {'ref_cnt': 0}) + return name def get_list(self, pool_name): @@ -87,6 +95,7 @@ class StorageVolumesModel(object): class StorageVolumeModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] + self.objstore = kargs['objstore'] def _get_storagevolume(self, pool, name): pool = StoragePoolModel.get_storagepool(pool, self.conn) @@ -101,16 +110,38 @@ class StorageVolumeModel(object): else: raise + def _get_ref_cnt(self, pool, name, path): + vol_id = '%s:%s' % (pool, name) + with self.objstore as session: + try: + ref_cnt = session.get('storagevolume', vol_id)['ref_cnt'] + except NotFoundError: + # Fix storage volume created outside kimchi scope + ref_cnt = 0 + args = {'conn': self.conn, 'objstore': self.objstore} + # try to find this volume in exsisted vm + vms = VMsModel(**args).get_list() + for vm in vms: + storages = VMStoragesModel(**args).get_list(vm) + for disk in storages: + if path == VMStorageModel(**args).lookup(vm, disk)['path']: + ref_cnt = ref_cnt + 1 + session.store('storagevolume', vol_id, {'ref_cnt': ref_cnt}) + + return ref_cnt + def lookup(self, pool, name): vol = self._get_storagevolume(pool, name) path = vol.path() info = vol.info() xml = vol.XMLDesc(0) fmt = xmlutils.xpath_get_text(xml, "/volume/target/format/@type")[0] + ref_cnt = self._get_ref_cnt(pool, name, path) res = dict(type=VOLUME_TYPE_MAP[info[0]], capacity=info[1], allocation=info[2], path=path, + ref_cnt=ref_cnt, format=fmt) if fmt == 'iso': if os.path.islink(path): -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Add mockmodel test and model test to validate: 1. ref_cnt of storage volume forked internal vm creation is 1. 2. ref_cnt of storage volume created from REST API is 0. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- tests/test_model.py | 4 ++++ tests/test_rest.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/tests/test_model.py b/tests/test_model.py index a08b3d9..e2ed273 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -353,6 +353,7 @@ class ModelTests(unittest.TestCase): inst.storagevolume_wipe(pool, vol) volinfo = inst.storagevolume_lookup(pool, vol) self.assertEquals(0, volinfo['allocation']) + self.assertEquals(0, volinfo['ref_cnt']) volinfo = inst.storagevolume_lookup(pool, vol) # Define the size = capacity + 16M @@ -402,6 +403,9 @@ class ModelTests(unittest.TestCase): vm_info = inst.vm_lookup(params['name']) disk_path = '/tmp/kimchi-images/%s-0.img' % vm_info['uuid'] self.assertTrue(os.access(disk_path, os.F_OK)) + vol = '%s-0.img' % vm_info['uuid'] + volinfo = inst.storagevolume_lookup(pool, vol) + self.assertEquals(1, volinfo['ref_cnt']) # reset template to default storage pool # so we can remove the storage pool created 'test-pool' diff --git a/tests/test_rest.py b/tests/test_rest.py index 54530f3..5deda27 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -244,6 +244,7 @@ class RestTests(unittest.TestCase): resp = self.request(vol_uri) vol = json.loads(resp.read()) self.assertEquals(1 << 30, vol['capacity']) + self.assertEquals(1, vol['ref_cnt']) # Start the VM resp = self.request('/vms/test-vm/start', '{}', 'POST') @@ -844,6 +845,7 @@ class RestTests(unittest.TestCase): storagevolume = json.loads(resp.read()) self.assertEquals('volume-1', storagevolume['name']) self.assertEquals('raw', storagevolume['format']) + self.assertEquals(0, storagevolume['ref_cnt']) self.assertEquals('/var/lib/libvirt/images/volume-1', storagevolume['path']) -- 1.8.1.2
From: Royce Lv <lvroyce@linux.vnet.ibm.com> These can be diagnosed by pep8 1.3.3, but may not report error in some version, These errors does not comply: http://www.python.org/dev/peps/pep-0008/ so fix them. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/isoinfo.py | 4 ++-- src/kimchi/mockmodel.py | 7 ++++--- src/kimchi/model/debugreports.py | 2 +- src/kimchi/model/storageservers.py | 4 ++-- src/kimchi/model/storagevolumes.py | 3 ++- src/kimchi/model/templates.py | 2 +- tests/test_model.py | 6 +++--- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/kimchi/isoinfo.py b/src/kimchi/isoinfo.py index 1e9e02d..8d423ad 100644 --- a/src/kimchi/isoinfo.py +++ b/src/kimchi/isoinfo.py @@ -183,7 +183,7 @@ class IsoImage(object): for i in xrange(1, 4): fmt = IsoImage.EL_TORITO_BOOT_RECORD ptr = i * IsoImage.SECTOR_SIZE - tmp_data = data[ptr:ptr+fmt.size] + tmp_data = data[ptr:ptr + fmt.size] if len(tmp_data) < fmt.size: return @@ -212,7 +212,7 @@ class IsoImage(object): {'filename': self.path}) fmt = IsoImage.EL_TORITO_BOOT_ENTRY - tmp_data = data[ptr:ptr+fmt.size] + tmp_data = data[ptr:ptr + fmt.size] (boot, media_type, load_seg, sys_type, pad0, sectors, load_rba) = self._unpack(fmt, tmp_data) if boot == 0x88: diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py index d69a58a..d909b14 100644 --- a/src/kimchi/mockmodel.py +++ b/src/kimchi/mockmodel.py @@ -220,7 +220,7 @@ class MockModel(object): if not name: iso = params['cdrom'] iso_name = os.path.splitext(iso[iso.rfind('/') + 1:])[0] - name = iso_name + str(int(time.time()*1000)) + name = iso_name + str(int(time.time() * 1000)) params['name'] = name if name in self._mock_templates: @@ -915,7 +915,8 @@ class MockVMIface(object): @classmethod def get_mac(cls): - mac = ":".join(["52", "54"] + ["%02x" % (cls.counter/(256**i) % 256) + mac = ":".join(["52", "54"] + + ["%02x" % (cls.counter / (256 ** i) % 256) for i in range(3, -1, -1)]) return mac @@ -1042,7 +1043,7 @@ class MockVMScreenshot(VMScreenshot): self.coord = (self.coord[0], self.coord[1], min(MockVMScreenshot.BOX_COORD[2], - self.coord[2]+random.randrange(50)), + self.coord[2] + random.randrange(50)), self.coord[3]) image = Image.new("RGB", (256, 256), self.background) diff --git a/src/kimchi/model/debugreports.py b/src/kimchi/model/debugreports.py index 3e14848..c6e698b 100644 --- a/src/kimchi/model/debugreports.py +++ b/src/kimchi/model/debugreports.py @@ -41,7 +41,7 @@ class DebugReportsModel(object): ident = params.get('name').strip() # Generate a name with time and millisec precision, if necessary if ident is None or ident == "": - ident = 'report-' + str(int(time.time()*1000)) + ident = 'report-' + str(int(time.time() * 1000)) taskid = self._gen_debugreport_file(ident) return self.task.lookup(taskid) diff --git a/src/kimchi/model/storageservers.py b/src/kimchi/model/storageservers.py index d4fba9d..167fedd 100644 --- a/src/kimchi/model/storageservers.py +++ b/src/kimchi/model/storageservers.py @@ -64,8 +64,8 @@ class StorageServerModel(object): for pool in pools: try: pool_info = self.pool.lookup(pool) - if pool_info['source'] and \ - pool_info['source']['addr'] == server: + if (pool_info['source'] and + pool_info['source']['addr'] == server): return dict(host=server) except NotFoundError: # Avoid inconsistent pool result because of lease between list diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index e9ce736..ec569ca 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -124,7 +124,8 @@ class StorageVolumeModel(object): for vm in vms: storages = VMStoragesModel(**args).get_list(vm) for disk in storages: - if path == VMStorageModel(**args).lookup(vm, disk)['path']: + d_info = VMStorageModel(**args).lookup(vm, disk) + if path == d_info['path']: ref_cnt = ref_cnt + 1 session.store('storagevolume', vol_id, {'ref_cnt': ref_cnt}) diff --git a/src/kimchi/model/templates.py b/src/kimchi/model/templates.py index 3062bd7..145e643 100644 --- a/src/kimchi/model/templates.py +++ b/src/kimchi/model/templates.py @@ -39,7 +39,7 @@ class TemplatesModel(object): if not name: iso = params['cdrom'] iso_name = os.path.splitext(iso[iso.rfind('/') + 1:])[0] - name = iso_name + str(int(time.time()*1000)) + name = iso_name + str(int(time.time() * 1000)) params['name'] = name conn = self.conn.get() diff --git a/tests/test_model.py b/tests/test_model.py index e2ed273..8499056 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -975,7 +975,7 @@ class ModelTests(unittest.TestCase): for repo in test_repos: inst.repositories_create(repo) host_repos = inst.repositories_get_list() - self.assertEquals(system_host_repos+len(test_repos), len(host_repos)) + self.assertEquals(system_host_repos + len(test_repos), len(host_repos)) for repo in test_repos: repo_info = inst.repository_lookup(repo.get('repo_id')) @@ -1013,7 +1013,7 @@ class ModelTests(unittest.TestCase): inst.repositories_create(repo) host_repos = inst.repositories_get_list() - self.assertEquals(system_host_repos+1, len(host_repos)) + self.assertEquals(system_host_repos + 1, len(host_repos)) new_repo = {'repo_id': 'fedora-fake', 'repo_name': 'Fedora 19 Update FAKE', @@ -1043,7 +1043,7 @@ class ModelTests(unittest.TestCase): inst.repositories_create(repo) host_repos = inst.repositories_get_list() - self.assertEquals(system_host_repos+1, len(host_repos)) + self.assertEquals(system_host_repos + 1, len(host_repos)) repo_info = inst.repository_lookup(repo.get('repo_id')) self.assertEquals(True, repo_info.get('enabled')) -- 1.8.1.2
Please, rebase and send it again. On 03/04/2014 04:38 AM, lvroyce@linux.vnet.ibm.com wrote:
From: Royce Lv <lvroyce@linux.vnet.ibm.com>
tested: 1. sudo make check 2. from UI: list storage volumes and unrefered volume ref_cnt is 0, ref_cnt for refered ones are 1.
Available volumes will be tracked by volume reference count. Add this field to storage volume, so that storage volume/pool action validation can rely on it. Royce Lv (6): Fix vm disk path when it does not have source element Add volume ref_cnt: update api.md Add volume ref_cnt: Update controller and json schema Add volume ref_cnt: Add model and mockmodel implementation Add volume ref_cnt: Update test Multiple pep8 fixes
docs/API.md | 3 +++ src/kimchi/API.json | 25 +++++++++++++++++++++++++ src/kimchi/control/storagevolumes.py | 1 + src/kimchi/i18n.py | 4 ++++ src/kimchi/isoinfo.py | 4 ++-- src/kimchi/mockmodel.py | 10 +++++++--- src/kimchi/model/debugreports.py | 2 +- src/kimchi/model/storageservers.py | 4 ++-- src/kimchi/model/storagevolumes.py | 32 ++++++++++++++++++++++++++++++++ src/kimchi/model/templates.py | 2 +- src/kimchi/model/vmstorages.py | 23 +++++++++++++---------- tests/test_model.py | 10 +++++++--- tests/test_rest.py | 2 ++ 13 files changed, 100 insertions(+), 22 deletions(-)
participants (2)
-
Aline Manera -
lvroyce＠linux.vnet.ibm.com