[PATCH v5 0/7] Support image upload
I'm sending a new version of Royce's patchset because she's currently on vacation. The changelog from the previous patchset (v4) is: - Use the same file body size in "src/kimchi.config" as in "src/kimchi.conf". - Make sure the file handler will be closed when reading the upload file content. - Fix the Task status message describing the upload progress. - Remove the pool refresh operation from the upload loop. - Add the package "python-requests" as a project requirement. - Fix a formatting issue. Royce Lv (7): Storage volume upload: Update API.md Fix mockmodel reset for objectstore Storage volume upload: Parse params for upload formdata Storage volume upload: Control request body size of kimchi Storage volume upload: Support file based upload Storage volume upload: Adding progress to task message Storage volume upload: Change mockmodel and test contrib/DEBIAN/control.in | 3 ++- contrib/kimchi.spec.fedora.in | 1 + contrib/kimchi.spec.suse.in | 1 + docs/API.md | 2 ++ docs/README.md | 7 ++++--- src/kimchi.conf.in | 3 +++ src/kimchi/config.py.in | 1 + src/kimchi/control/utils.py | 4 +++- src/kimchi/mockmodel.py | 24 ++++++++++++++++++++++++ src/kimchi/model/storagevolumes.py | 31 +++++++++++++++++++++++++++++++ src/kimchi/server.py | 1 + src/kimchid.in | 2 ++ tests/test_rest.py | 38 +++++++++++++++++++++++++++++++++++++- tests/utils.py | 2 +- 14 files changed, 113 insertions(+), 7 deletions(-) -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Update API.md adding upload option to storagevolume creation. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- docs/API.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/API.md b/docs/API.md index 9b1bff3..c070fa7 100644 --- a/docs/API.md +++ b/docs/API.md @@ -413,11 +413,13 @@ A interface represents available network interface on VM. in the defined Storage Pool * **POST**: Create a new Storage Volume in the Storage Pool The return resource is a task resource * See Resource: Task * + Only one of 'file', 'capacity', 'url' can be specified. * name: The name of the Storage Volume * type: The type of the defined Storage Volume * capacity: The total space which can be used to store volumes The unit is MBytes * format: The format of the defined Storage Volume + * file: File to be uploaded, passed through form data * url: URL to be downloaded ### Resource: Storage Volume -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> In mockmodel reset, we only reset model fields dict but objectstore remains unchanged, tasks retained in objecstore making mockmodel task related tests fail. Reset objectstore to fix this. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/mockmodel.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py index 908b0b7..41e001c 100644 --- a/src/kimchi/mockmodel.py +++ b/src/kimchi/mockmodel.py @@ -64,6 +64,7 @@ class MockModel(object): def __init__(self, objstore_loc=None): self.reset() self.objstore = ObjectStore(objstore_loc) + self.objstore_loc = objstore_loc self.distros = self._get_distros() def capabilities_lookup(self, *ident): @@ -78,6 +79,8 @@ class MockModel(object): 'federation': 'off'} def reset(self): + if hasattr(self, 'objstore'): + self.objstore = ObjectStore(self.objstore_loc) self._mock_vms = {} self._mock_screenshots = {} self._mock_templates = {} -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> When pass form to cherrypy, it is stored in cherrypy.request.params, so adjust parse_request for this change. Also avoid body parse so that file content will not be loaded into mem. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> --- src/kimchi/control/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/kimchi/control/utils.py b/src/kimchi/control/utils.py index fbd5177..64e0ac5 100644 --- a/src/kimchi/control/utils.py +++ b/src/kimchi/control/utils.py @@ -72,14 +72,16 @@ def mime_in_header(header, mime): def parse_request(): if 'Content-Length' not in cherrypy.request.headers: return {} - rawbody = cherrypy.request.body.read() if mime_in_header('Content-Type', 'application/json'): + rawbody = cherrypy.request.body.read() try: return json.loads(rawbody) except ValueError: e = OperationFailed('KCHAPI0006E') raise cherrypy.HTTPError(400, e.message) + elif mime_in_header('Content-Type', 'multipart/form-data'): + return cherrypy.request.params else: e = OperationFailed('KCHAPI0007E') raise cherrypy.HTTPError(415, e.message) -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> This patch control request body size to be 4G, when request size exceed, kimchi will response with 413. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- src/kimchi.conf.in | 3 +++ src/kimchi/config.py.in | 1 + src/kimchi/server.py | 1 + src/kimchid.in | 2 ++ 4 files changed, 7 insertions(+) diff --git a/src/kimchi.conf.in b/src/kimchi.conf.in index 2be1a0e..ea39292 100644 --- a/src/kimchi.conf.in +++ b/src/kimchi.conf.in @@ -30,6 +30,9 @@ # at the same network #federation = off +# Max request body size in KB, default value is 4GB +#max_body_size = 4 * 1024 * 1024 + [logging] # Log directory #log_dir = @localstatedir@/log/kimchi diff --git a/src/kimchi/config.py.in b/src/kimchi/config.py.in index d403827..91e5f48 100644 --- a/src/kimchi/config.py.in +++ b/src/kimchi/config.py.in @@ -255,6 +255,7 @@ def _get_config(): config.set("server", "ssl_key", "") config.set("server", "environment", "production") config.set("server", "federation", "off") + config.set('server', 'max_body_size', '4*1024*1024') config.add_section("logging") config.set("logging", "log_dir", paths.log_dir) config.set("logging", "log_level", DEFAULT_LOG_LEVEL) diff --git a/src/kimchi/server.py b/src/kimchi/server.py index 10f5dff..1c3b360 100644 --- a/src/kimchi/server.py +++ b/src/kimchi/server.py @@ -87,6 +87,7 @@ class Server(object): # directly. You must go through the proxy. cherrypy.server.socket_host = '127.0.0.1' cherrypy.server.socket_port = options.cherrypy_port + cherrypy.server.max_request_body_size = eval(options.max_body_size) cherrypy.log.screen = True cherrypy.log.access_file = options.access_log diff --git a/src/kimchid.in b/src/kimchid.in index 3ed087f..075b744 100644 --- a/src/kimchid.in +++ b/src/kimchid.in @@ -86,6 +86,8 @@ def main(options): # Add non-option arguments setattr(options, 'ssl_cert', config.config.get('server', 'ssl_cert')) setattr(options, 'ssl_key', config.config.get('server', 'ssl_key')) + setattr(options, 'max_body_size', + config.config.get('server', 'max_body_size')*1024) kimchi.server.main(options) -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Start a task to upload storage volume. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- src/kimchi/model/storagevolumes.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index 6037953..7a188d2 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -83,6 +83,33 @@ class StorageVolumesModel(object): taskid = add_task(targeturi, create_func, self.objstore, params) return self.task.lookup(taskid) + def _create_volume_with_file(self, cb, params): + pool_name = params.pop('pool') + dir_path = StoragePoolModel( + conn=self.conn, objstore=self.objstore).lookup(pool_name)['path'] + file_path = os.path.join(dir_path, params['name']) + if os.path.exists(file_path): + raise InvalidParameter('KCHVOL0001E', {'name': params['name']}) + + upload_file = params['file'] + try: + with open(file_path, 'wb') as f: + while True: + data = upload_file.file.read(8192) + if not data: + break + f.write(data) + except Exception as e: + raise OperationFailed('KCHVOL0007E', + {'name': params['name'], + 'pool': pool_name, + 'err': e.message}) + + # Refresh to make sure volume can be found in following lookup + pool = StoragePoolModel.get_storagepool(pool_name, self.conn) + pool.refresh() + cb('OK', True) + def _create_volume_with_capacity(self, cb, params): pool_name = params.pop('pool') vol_xml = """ -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Periodically update current written bytes to task message so that UI can query the latest progress of volume uploading. Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- src/kimchi/model/storagevolumes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/kimchi/model/storagevolumes.py b/src/kimchi/model/storagevolumes.py index 7a188d2..ef8e684 100644 --- a/src/kimchi/model/storagevolumes.py +++ b/src/kimchi/model/storagevolumes.py @@ -92,13 +92,17 @@ class StorageVolumesModel(object): raise InvalidParameter('KCHVOL0001E', {'name': params['name']}) upload_file = params['file'] + f_len = upload_file.fp.length try: + size = 0 with open(file_path, 'wb') as f: while True: data = upload_file.file.read(8192) if not data: break + size += len(data) f.write(data) + cb('%s/%s' % (size, f_len)) except Exception as e: raise OperationFailed('KCHVOL0007E', {'name': params['name'], -- 1.9.3
From: Royce Lv <lvroyce@linux.vnet.ibm.com> Change mockmodel and tests to support storagevolume upload. Also, add the package "python-requests" as a dependency Signed-off-by: Royce Lv <lvroyce@linux.vnet.ibm.com> Signed-off-by: Crístian Viana <vianac@linux.vnet.ibm.com> --- contrib/DEBIAN/control.in | 3 ++- contrib/kimchi.spec.fedora.in | 1 + contrib/kimchi.spec.suse.in | 1 + docs/README.md | 7 ++++--- src/kimchi/mockmodel.py | 21 +++++++++++++++++++++ tests/test_rest.py | 38 +++++++++++++++++++++++++++++++++++++- tests/utils.py | 2 +- 7 files changed, 67 insertions(+), 6 deletions(-) diff --git a/contrib/DEBIAN/control.in b/contrib/DEBIAN/control.in index 3754fa2..99546fa 100644 --- a/contrib/DEBIAN/control.in +++ b/contrib/DEBIAN/control.in @@ -27,6 +27,7 @@ Depends: python-cherrypy3 (>= 3.2.0), python-guestfs, libguestfs-tools Build-Depends: libxslt, - python-libxml2 + python-libxml2, + python-requests Maintainer: Aline Manera <alinefm@br.ibm.com> Description: Kimchi web server diff --git a/contrib/kimchi.spec.fedora.in b/contrib/kimchi.spec.fedora.in index 5766784..bd1f04d 100644 --- a/contrib/kimchi.spec.fedora.in +++ b/contrib/kimchi.spec.fedora.in @@ -31,6 +31,7 @@ Requires: python-libguestfs Requires: libguestfs-tools BuildRequires: libxslt BuildRequires: libxml2-python +BuildRequires: python-requests %if 0%{?fedora} >= 15 || 0%{?rhel} >= 7 %global with_systemd 1 diff --git a/contrib/kimchi.spec.suse.in b/contrib/kimchi.spec.suse.in index 1f193d0..d0bf317 100644 --- a/contrib/kimchi.spec.suse.in +++ b/contrib/kimchi.spec.suse.in @@ -30,6 +30,7 @@ Requires: python-libguestfs Requires: guestfs-tools BuildRequires: libxslt-tools BuildRequires: python-libxml2 +BuildRequires: python-requests %if 0%{?sles_version} == 11 Requires: python-ordereddict diff --git a/docs/README.md b/docs/README.md index 24537e1..3bdc914 100644 --- a/docs/README.md +++ b/docs/README.md @@ -55,7 +55,7 @@ Install Dependencies python-ipaddr python-lxml nfs-utils \ iscsi-initiator-utils libxslt pyparted nginx \ policycoreutils-python python-libguestfs \ - libguestfs-tools + libguestfs-tools python-requests # If using RHEL6, install the following additional packages: $ sudo yum install python-unittest2 python-ordereddict # Restart libvirt to allow configuration changes to take effect @@ -78,7 +78,8 @@ for more information on how to configure your system to access this repository. qemu-kvm libtool python-psutil python-ethtool \ sosreport python-ipaddr python-lxml nfs-common \ open-iscsi lvm2 xsltproc python-parted nginx \ - firewalld python-guestfs libguestfs-tools + firewalld python-guestfs libguestfs-tools \ + python-requests Packages version requirement: python-jsonschema >= 1.3.0 @@ -93,7 +94,7 @@ for more information on how to configure your system to access this repository. rpm-build kvm python-psutil python-ethtool \ python-ipaddr python-lxml nfs-client open-iscsi \ libxslt-tools python-xml python-parted \ - python-libguestfs guestfs-tools + python-libguestfs guestfs-tools python-requests Packages version requirement: python-psutil >= 0.6.0 diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py index 41e001c..15a75af 100644 --- a/src/kimchi/mockmodel.py +++ b/src/kimchi/mockmodel.py @@ -512,6 +512,27 @@ class MockModel(object): taskid = self.add_task(targeturi, create_func, params) return self.task_lookup(taskid) + def _create_volume_with_file(self, cb, params): + upload_file = params['file'] + params['name'] = params['name'] + params['format'] = 'raw' + params['capacity'] = upload_file.fp.length + size = 0 + try: + while True: + data = upload_file.file.read(8192) + if not data: + break + size += len(data) + cb('%s/%s' % (size, params['capacity']), True) + except Exception as e: + raise OperationFailed('KCHVOL0007E', + {'name': params['name'], + 'pool': params['pool'], + 'err': e.message}) + self._create_volume_with_capacity(cb, params) + cb('%s/%s' % (size, params['capacity']), True) + def _create_volume_with_capacity(self, cb, params): pool_name = params.pop('pool') pool = self._get_storagepool(pool_name) diff --git a/tests/test_rest.py b/tests/test_rest.py index 567a954..ae1c971 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -22,6 +22,7 @@ import base64 import json import os import random +import requests import shutil import time import unittest @@ -1058,7 +1059,10 @@ class RestTests(unittest.TestCase): resp = self.request('/storagepools/pool-2/storagevolumes/', req, 'POST') self.assertEquals(202, resp.status) - time.sleep(1) + task_id = json.loads(resp.read())['id'] + self._wait_task(task_id) + status = json.loads(self.request('/tasks/%s' % task_id).read()) + self.assertEquals('finished', status['status']) # Verify the storage volume resp = self.request('/storagepools/pool-2/storagevolumes/test-volume') @@ -1892,6 +1896,38 @@ class RestTests(unittest.TestCase): resp = self.request('%s/fedora-fake' % base_uri, '{}', 'DELETE') self.assertEquals(204, resp.status) + def test_upload(self): + # If we use self.request, we may encode multipart formdata by ourselves + # requests lib take care of encode part, so use this lib instead + def fake_auth_header(): + headers = {'Accept': 'application/json'} + user, pw = kimchi.mockmodel.fake_user.items()[0] + hdr = "Basic " + base64.b64encode("%s:%s" % (user, pw)) + headers['AUTHORIZATION'] = hdr + return headers + + with RollbackContext() as rollback: + upload_path = '/tmp/vol' + f = open(upload_path, 'w') + f.write('abcd') + f.close() + rollback.prependDefer(os.remove, upload_path) + + vol = open(upload_path, 'rb') + url = "https://%s:%s/storagepools/default/storagevolumes" %\ + (host, ssl_port) + + r = requests.post(url, + files={'file': vol}, + data={'name': 'new_vol2'}, + verify=False, + headers=fake_auth_header()) + self.assertEquals(r.status_code, 202) + time.sleep(1) + vol_list = json.loads( + self.request('/storagepools/default/storagevolumes').read()) + self.assertEquals('new_vol2', vol_list[0]['name']) + class HttpsRestTests(RestTests): """ diff --git a/tests/utils.py b/tests/utils.py index 7fccae1..140bb1d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -104,7 +104,7 @@ def run_server(host, port, ssl_port, test_mode, cherrypy_port=None, args = type('_', (object,), {'host': host, 'port': port, 'ssl_port': ssl_port, - 'cherrypy_port': cherrypy_port, + 'cherrypy_port': cherrypy_port, 'max_body_size': '4*1024', 'ssl_cert': '', 'ssl_key': '', 'test': test_mode, 'access_log': '/dev/null', 'error_log': '/dev/null', 'environment': environment, -- 1.9.3
participants (2)
-
Aline Manera -
Crístian Viana