[PATCH 0/4] WIP: Download remote image to storage pool
This patchset is a work in progress. I'm sending it now so we can get early feedback. The following problems are known: - The previous "create" function (which is now called "_create_volume_with_capacity") should be updated to the Task API. This will cause errors in existing tests. - There is one unfinished error message. - The tests and mockmodel are missing. Any feedback is welcome. Crístian Viana (4): storagevolume: Split "create" function based on the parameters storagevolume: Use a Task to create a new volume storagevolume: Download remote images to a storage pool storagevolume: Add download progress to task src/kimchi/control/storagevolumes.py | 4 +- src/kimchi/model/storagevolumes.py | 71 +++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) -- 1.9.3
The function "create" in StorageVolume creates a volume with only .... Soon, Kimchi will support creating storage volumes in different way, so we need to prepare for that change. Split the function based on its parameters and call the appropriate implementation. Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/model/storagevolumes.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index b60884c..2c3e7a8 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -44,6 +44,21 @@ class StorageVolumesModel(object): self.objstore = kargs['objstore'] def create(self, pool_name, params): + vol_source = ['file', 'url', 'capacity'] + + if len([p for p in params if p in vol_source]) != 1: + raise InvalidParameter('KCHVOL0018E', {'param': str(vol_source)}) + + for v in vol_source: + if v in params: + create_func = getattr(self, '_create_volume_with_' + v, None) + if not create_func: + # TODO: provide a better and internationalized error msg + raise OperationFailed('<internal error>') + + return create_func(pool_name, params) + + def _create_volume_with_capacity(self, pool_name, params): vol_xml = """ <volume> <name>%(name)s</name> -- 1.9.3
Creating a new storage volume may take a long time, especially for the download and upload features which will be implemented soon. Instead of waiting for the create operation to finish, return a Task so the client can fetch its status later. Use an AsyncTask to create new storage volumes. Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- src/kimchi/control/storagevolumes.py | 4 ++-- src/kimchi/model/storagevolumes.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/kimchi/control/storagevolumes.py b/src/kimchi/control/storagevolumes.py index 327bf75..79170ee 100644 --- a/src/kimchi/control/storagevolumes.py +++ b/src/kimchi/control/storagevolumes.py @@ -18,11 +18,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import kimchi.template -from kimchi.control.base import Collection, Resource +from kimchi.control.base import AsyncCollection, Collection, Resource from kimchi.control.utils import get_class_name, model_fn -class StorageVolumes(Collection): +class StorageVolumes(AsyncCollection): def __init__(self, model, pool): super(StorageVolumes, self).__init__(model) self.resource = StorageVolume diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index 2c3e7a8..bd9dbc9 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -28,6 +28,7 @@ from kimchi.exception import MissingParameter, NotFoundError, OperationFailed from kimchi.isoinfo import IsoImage from kimchi.model.storagepools import StoragePoolModel from kimchi.utils import kimchi_log +from kimchi.model.tasks import TaskModel from kimchi.model.vms import VMsModel, VMModel from kimchi.vmdisks import get_vm_disk, get_vm_disk_list @@ -42,6 +43,7 @@ class StorageVolumesModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] self.objstore = kargs['objstore'] + self.task = TaskModel(**kargs) def create(self, pool_name, params): vol_source = ['file', 'url', 'capacity'] @@ -58,6 +60,7 @@ class StorageVolumesModel(object): return create_func(pool_name, params) + # TODO: this function must be updated to return a Task def _create_volume_with_capacity(self, pool_name, params): vol_xml = """ <volume> -- 1.9.3
In order to simplify the creation of a remote image in Kimchi, the user will be able to provide a remote URL and the Kimchi server will download it and put it on a storage pool. Download a remote image to a storage pool Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- src/kimchi/model/storagevolumes.py | 49 +++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index bd9dbc9..dc19f4f 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -18,6 +18,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import os +import urllib2 import libvirt @@ -27,7 +28,7 @@ from kimchi.exception import InvalidOperation, InvalidParameter, IsoFormatError from kimchi.exception import MissingParameter, NotFoundError, OperationFailed from kimchi.isoinfo import IsoImage from kimchi.model.storagepools import StoragePoolModel -from kimchi.utils import kimchi_log +from kimchi.utils import add_task, kimchi_log from kimchi.model.tasks import TaskModel from kimchi.model.vms import VMsModel, VMModel from kimchi.vmdisks import get_vm_disk, get_vm_disk_list @@ -108,6 +109,52 @@ class StorageVolumesModel(object): return name + def _create_volume_with_url(self, pool_name, params): + def _download(cb, params): + pool = params['pool'] + target = params['target'] + url = params['url'] + + chunk_size = 4096 # 4 kiB + + response = volume = None + try: + response = urllib2.urlopen(url) + volume = open(target, 'w') + + while True: + chunk_data = response.read(chunk_size) + if not chunk_data: + break + + volume.write(chunk_data) + except Exception, e: + raise OperationFailed('KCHVOL0007E', {'name': name, + 'pool': pool_name, + 'err': e.message}) + finally: + if response is not None: + response.close() + + if volume is not None: + volume.close() + + cb('OK', True) + pool.refresh() + + name = params['name'] + url = params['url'] + + pool_model = StoragePoolModel(conn=self.conn, objstore=self.objstore) + pool = pool_model.lookup(pool_name) + + download_params = {'pool': StoragePoolModel.get_storagepool(pool_name, + self.conn), + 'target': os.path.join(pool['path'], name), + 'url': url} + taskid = add_task('', _download, self.objstore, download_params) + return self.task.lookup(taskid) + def get_list(self, pool_name): pool = StoragePoolModel.get_storagepool(pool_name, self.conn) if not pool.isActive(): -- 1.9.3
The download operation can take a long time, so it's nice to have some way to track its current progress. Update the task's message with the number of bytes downloaded. The format is "<bytes downloaded>/<total bytes>". If the number of bytes cannot be read from the remote URL, <total bytes> will be "-". Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- src/kimchi/model/storagevolumes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index dc19f4f..c044973 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -121,6 +121,8 @@ class StorageVolumesModel(object): try: response = urllib2.urlopen(url) volume = open(target, 'w') + remote_size = response.info().getheader('Content-Length', '-') + downloaded_size = 0 while True: chunk_data = response.read(chunk_size) @@ -128,6 +130,8 @@ class StorageVolumesModel(object): break volume.write(chunk_data) + downloaded_size += len(chunk_data) + cb('downloading %s/%s' % (downloaded_size, remote_size)) except Exception, e: raise OperationFailed('KCHVOL0007E', {'name': name, 'pool': pool_name, -- 1.9.3
Applied. Thanks. Regards, Aline Manera
participants (2)
-
Aline Manera -
Crístian Viana