April 2016 - Kimchi-devel - oVirt List Archives

[PATCH] [Kimchi] Fix template creation from image file
by Jose Ricardo Ziviani 08 Apr '16

08 Apr '16

- When using an existing image as base for creating templates we cannot rely on its pool because it's undefined. This commit sets the default pool for this case. Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com> --- vmtemplate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vmtemplate.py b/vmtemplate.py index 653ad02..b71cf23 100644 --- a/vmtemplate.py +++ b/vmtemplate.py @@ -96,7 +96,12 @@ class VMTemplate(object): for index, disk in enumerate(disks): disk_info = dict(default_disk) - pool_type = self._get_storage_type(disk['pool']['name']) + if 'pool' in disk and 'name' in disk['pool']: + pool_type = self._get_storage_type(disk['pool']['name']) + else: + pool_type = self._get_storage_type( + default_disk['pool']['name']) + if pool_type in ['iscsi', 'scsi']: disk_info = {'index': 0, 'format': 'raw', 'volume': None} -- 1.9.1

3 2

[PATCH] [Wok] Implement update methods for object store
by Jose Ricardo Ziviani 07 Apr '16

07 Apr '16

- This commit implements two new methods for object store, update_data and update_id. Now it's possible to update an existing data instead of delete/create. Signed-off-by: Jose Ricardo Ziviani <joserz(a)linux.vnet.ibm.com> --- src/wok/objectstore.py | 13 +++++++++++++ tests/test_objectstore.py | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/wok/objectstore.py b/src/wok/objectstore.py index ff3796c..5ec87eb 100644 --- a/src/wok/objectstore.py +++ b/src/wok/objectstore.py @@ -94,6 +94,19 @@ class ObjectStoreSession(object): (ident, obj_type, jsonstr, version)) self.conn.commit() + def update_data(self, obj_type, ident, data): + jsonstr = json.dumps(data) + c = self.conn.cursor() + c.execute('UPDATE objects SET json=? WHERE id=? AND type=?', + (jsonstr, ident, obj_type)) + self.conn.commit() + + def update_id(self, obj_type, ident, new_ident): + c = self.conn.cursor() + c.execute('UPDATE objects SET id=? WHERE id=? AND type=?', + (new_ident, ident, obj_type)) + self.conn.commit() + class ObjectStore(object): def __init__(self, location=None): diff --git a/tests/test_objectstore.py b/tests/test_objectstore.py index 3ea7b70..d6ea434 100644 --- a/tests/test_objectstore.py +++ b/tests/test_objectstore.py @@ -82,6 +82,17 @@ class ObjectStoreTests(unittest.TestCase): item = session.get_object_version('f��', 't��st1') self.assertEquals(get_version().split('-')[0], item[0]) + # test update data + session.update_data('f��', 't��st1', {'��': 5}) + item = session.get('f��', 't��st1') + self.assertEquals(5, item[u'��']) + + # test update id + session.update_id('f��', 't��st1', 't��st2') + item = session.get('f��', 't��st2') + self.assertEquals(5, item[u'��']) + + def test_object_store_threaded(self): def worker(ident): with store as session: -- 1.9.1

3 3

[PATCH v5][Kimchi 0/4] Use a single field to create a template
by Ramon Medeiros 07 Apr '16

07 Apr '16

Changes: v2: Remove libvirt connection from VMtemplate Fix incorrect changes on tests Fix return status when source_media not passed v3: Fix pep8 issues Remove/add some constants Rewrite API message v4: Remove cdrom verification on model Use disk info from osinfo when adding disk based template Ramon Medeiros (4): Create a single field to pass the installation media Fix checking duplicate template before creating it Identify installation media while creating template Update tests API.json | 9 ++-- i18n.py | 2 +- model/templates.py | 26 +++++------ tests/test_authorization.py | 4 +- tests/test_livemigration.py | 7 +-- tests/test_mockmodel.py | 13 +++--- tests/test_model.py | 94 +++++++++++++++++++++------------------ tests/test_rest.py | 36 ++++++++------- tests/test_template.py | 47 +++++++++----------- tests/test_vmtemplate.py | 40 +++++++++-------- vmtemplate.py | 104 +++++++++++++++++++++++++++++++++++++------- 11 files changed, 227 insertions(+), 155 deletions(-) -- 2.5.5

3 12

[PATCH V2] [Kimchi] Add support to Libvirt Events.
by pvital＠linux.vnet.ibm.com 07 Apr '16

07 Apr '16

From: Paulo Vital <pvital(a)linux.vnet.ibm.com> This patch adds support to handle in Kimchi any Libvirt Event, just by adding a callback to process the event and register the event with the callback. A method called register_common_events() is responsible to register the common domain events to use a generic callback that logs into error log, the event happened and it's details. This patch is part of the solution for Kimchi Issue #817 Signed-off-by: Paulo Vital <pvital(a)linux.vnet.ibm.com> --- i18n.py | 4 +++ model/libvirtevents.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ model/model.py | 6 +++- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 model/libvirtevents.py diff --git a/i18n.py b/i18n.py index 6214687..3939aba 100644 --- a/i18n.py +++ b/i18n.py @@ -326,4 +326,8 @@ messages = { "KCHLVMS0001E": _("Invalid volume group name parameter: %(name)s."), + "KCHEVENT0001E": _("Failed to register the default event implementation."), + "KCHEVENT0002E": _("Failed to register timeout event."), + "KCHEVENT0003E": _("Failed to Run the default event implementation."), + } diff --git a/model/libvirtevents.py b/model/libvirtevents.py new file mode 100644 index 0000000..2775476 --- /dev/null +++ b/model/libvirtevents.py @@ -0,0 +1,97 @@ +# +# Project Kimchi +# +# Copyright IBM, Corp. 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import cherrypy +import libvirt +import time + +from wok.exception import OperationFailed +from wok.utils import wok_log + + +class LibvirtEvents(object): + def __init__(self): + # Register default implementation of event handlers + if libvirt.virEventRegisterDefaultImpl() < 0: + raise OperationFailed('KCHEVENT0001E') + + # Run a background thread with the event loop. Using cherrypy + # BackgroundTask class due to issues when using threading module with + # cherrypy. + self.event_loop_thread = cherrypy.process.plugins.BackgroundTask( + 2, + self._event_loop_run + ) + self.event_loop_thread.setName('KimchiLibvirtEventLoop') + self.event_loop_thread.setDaemon(True) + self.event_loop_thread.start() + + # Set an event timeout to control the self._event_loop_run + if libvirt.virEventAddTimeout(0, self._kimchi_EventTimeout, None) < 0: + raise OperationFailed('KCHEVENT0002E') + + # Event loop method to be executed in background as thread + def _event_loop_run(self): + while True: + if libvirt.virEventRunDefaultImpl() < 0: + raise OperationFailed('KCHEVENT0003E') + + def is_event_loop_alive(self): + return self.event_loop_thread.isAlive() + + # Event loop handler used to limit length of waiting for any other event. + def _kimchi_EventTimeout(self, timer, opaque): + time.sleep(1) + + def domain_event_generic_cb(self, conn, dom, event, detail, *args): + """ + Generic callback to handle Domain (VMs) events. + """ + evStrings = ("Defined", "Undefined", "Started", "Suspended", "Resumed", + "Stopped", "Shutdown", "PMSuspended", "Crashed") + evDetails = (("Added", "Updated"), + ("Removed", ), + ("Booted", "Migrated", "Restored", "Snapshot", "Wakeup"), + ("Paused", "Migrated", "IOError", "Watchdog", "Restored", + "Snapshot", "API error"), + ("Unpaused", "Migrated", "Snapshot"), + ("Shutdown", "Destroyed", "Crashed", "Migrated", "Saved", + "Failed", "Snapshot"), + ("Finished", ), + ("Memory", "Disk"), + ("Panicked")) + msg = "Libvirt Event: Domain %s %s %s" % (dom.name(), evStrings[event], + evDetails[event][detail]) + + wok_log.error(msg) + + def register_common_domain_events(self, conn): + """ + Register the most common Libvirt domain events to be handled. + """ + conn = conn.get() + for ev in (libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE, + libvirt.VIR_DOMAIN_EVENT_ID_REBOOT, + libvirt.VIR_DOMAIN_EVENT_ID_RTC_CHANGE, + libvirt.VIR_DOMAIN_EVENT_ID_IO_ERROR_REASON, + libvirt.VIR_DOMAIN_EVENT_ID_GRAPHICS, + libvirt.VIR_DOMAIN_EVENT_ID_BLOCK_JOB, + libvirt.VIR_DOMAIN_EVENT_ID_WATCHDOG): + conn.domainEventRegisterAny(None, ev, self.domain_event_generic_cb, + ev) diff --git a/model/model.py b/model/model.py index e44f804..aeb8b7d 100644 --- a/model/model.py +++ b/model/model.py @@ -26,6 +26,7 @@ from wok.plugins.kimchi import config from wok.utils import import_module, listPathModules from wok.plugins.kimchi.model.libvirtconnection import LibvirtConnection +from wok.plugins.kimchi.model.libvirtevents import LibvirtEvents class Model(BaseModel): @@ -43,8 +44,11 @@ class Model(BaseModel): return instances self.objstore = ObjectStore(objstore_loc or config.get_object_store()) + self.events = LibvirtEvents() self.conn = LibvirtConnection(libvirt_uri) - kargs = {'objstore': self.objstore, 'conn': self.conn} + self.events.register_common_domain_events(self.conn) + kargs = {'objstore': self.objstore, 'conn': self.conn, + 'eventsloop': self.events} models = [] # Import task model from Wok -- 2.5.5

3 2

[PATCH v4][Kimchi 0/4] Use a single field to create a template
by Ramon Medeiros 06 Apr '16

06 Apr '16

Instead of specify if the media is cdrom or disk, use source_media to create a template Changes: v2: Remove libvirt connection from VMtemplate Fix incorrect changes on tests Fix return status when source_media not passed v3: Fix pep8 issues Remove/add some constants Rewrite API message Ramon Medeiros (4): Create a single field to pass the installation media Fix checking duplicate template before creating it Identify installation media while creating template Update tests API.json | 5 ++ i18n.py | 2 +- model/templates.py | 15 ++++-- tests/test_authorization.py | 4 +- tests/test_livemigration.py | 7 +-- tests/test_mockmodel.py | 13 +++--- tests/test_model.py | 94 ++++++++++++++++++++----------------- tests/test_rest.py | 36 +++++++-------- tests/test_template.py | 47 +++++++++---------- tests/test_vmtemplate.py | 40 ++++++++-------- vmtemplate.py | 110 +++++++++++++++++++++++++++++++++++++------- 11 files changed, 235 insertions(+), 138 deletions(-) -- 2.5.5

3 16

[PATCH] [Kimchi] Bug fix: Use 256MB of memory on feature tests due restriction on Power machines
by Aline Manera 06 Apr '16

06 Apr '16

On Power machines the amount of memory must be aligned with 256MB. So adjust all feature tests according to it to avoid issues. Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com> --- model/featuretests.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model/featuretests.py b/model/featuretests.py index ba728e1..23e7eeb 100644 --- a/model/featuretests.py +++ b/model/featuretests.py @@ -34,7 +34,7 @@ FEATURETEST_POOL_NAME = "FEATURETEST_POOL" ISO_STREAM_XML = """ <domain type='%(domain)s'> <name>%(name)s</name> - <memory unit='KiB'>1048576</memory> + <memory unit='MiB'>256</memory> <os> <type arch='%(arch)s'>hvm</type> <boot dev='cdrom'/> @@ -56,7 +56,7 @@ ISO_STREAM_XML = """ SIMPLE_VM_XML = """ <domain type='%(domain)s'> <name>%(name)s</name> - <memory unit='KiB'>10240</memory> + <memory unit='MiB'>256</memory> <os> <type arch='%(arch)s'>hvm</type> <boot dev='hd'/> @@ -66,15 +66,15 @@ SIMPLE_VM_XML = """ MAXMEM_VM_XML = """ <domain type='%(domain)s'> <name>%(name)s</name> - <maxMemory slots='1' unit='KiB'>20480</maxMemory> - <memory unit='KiB'>10240</memory> + <maxMemory slots='1' unit='MiB'>512</maxMemory> + <memory unit='MiB'>256</memory> <os> <type arch='%(arch)s'>hvm</type> <boot dev='hd'/> </os> <cpu> <numa> - <cell id='0' cpus='0' memory='10240' unit='KiB'/> + <cell id='0' cpus='0' memory='256' unit='MiB'/> </numa> </cpu> <features> @@ -85,7 +85,7 @@ MAXMEM_VM_XML = """ DEV_MEM_XML = """ <memory model='dimm'> <target> - <size unit='KiB'>10240</size> + <size unit='MiB'>256</size> <node>0</node> </target> </memory>""" -- 2.5.5

2 2

[PATCH V2] [Wok] Fix errors in Ginger/Gingerbase/Gingers390x tests
by Lucio Correia 04 Apr '16

04 Apr '16

Do not try to translate message when app is not found. Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com> --- src/wok/message.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) Changes in V2: applied review. diff --git a/src/wok/message.py b/src/wok/message.py index 7ea5711..a3794d0 100644 --- a/src/wok/message.py +++ b/src/wok/message.py @@ -45,11 +45,11 @@ class WokMessage(object): self.plugin = plugin def _get_translation(self): - wok_app = cherrypy.tree.apps[''] + wok_app = cherrypy.tree.apps.get('', None) # get app from plugin path if specified if self.plugin: - app = cherrypy.tree.apps[self.plugin] + app = cherrypy.tree.apps.get(self.plugin, None) # if on request, try to get app from it elif cherrypy.request.app: app = cherrypy.request.app @@ -57,9 +57,12 @@ class WokMessage(object): else: app = wok_app + if app is None: + return self.code + # fallback to Wok message in case plugins raise Wok exceptions - text = app.root.messages.get(self.code, None) - if text is None: + text = app.root.messages.get(self.code, self.code) + if text == self.code and wok_app is not None: app = wok_app text = app.root.messages.get(self.code, self.code) -- 1.9.1

3 2

[PATCH] [Wok] Fix errors in Ginger/Gingerbase/Gingers390x tests
by Lucio Correia 04 Apr '16

04 Apr '16

Do not try to translate message when app is not found. Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com> --- src/wok/message.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/wok/message.py b/src/wok/message.py index 7ea5711..e4ee640 100644 --- a/src/wok/message.py +++ b/src/wok/message.py @@ -45,11 +45,11 @@ class WokMessage(object): self.plugin = plugin def _get_translation(self): - wok_app = cherrypy.tree.apps[''] + wok_app = cherrypy.tree.apps.get('', None) # get app from plugin path if specified if self.plugin: - app = cherrypy.tree.apps[self.plugin] + app = cherrypy.tree.apps.get(self.plugin, None) # if on request, try to get app from it elif cherrypy.request.app: app = cherrypy.request.app @@ -57,11 +57,13 @@ class WokMessage(object): else: app = wok_app + if app is None: + return self.code + # fallback to Wok message in case plugins raise Wok exceptions - text = app.root.messages.get(self.code, None) - if text is None: - app = wok_app - text = app.root.messages.get(self.code, self.code) + text = app.root.messages.get(self.code, self.code) + if text == self.code and wok_app is not None: + text = wok_app.root.messages.get(self.code, self.code) # do translation domain = app.root.domain -- 1.9.1

2 1

[PATCH V3] [Wok] Implement Asynchronous Notifications backend
by Lucio Correia 04 Apr '16

04 Apr '16

* There is no POST method: a notification is added through add_notification() method. * Notifications are always stored in Wok object store under 'notification' type, even if added by a plugin. * Every time Wok is started, all notifications are erased, since this is a UI feature, intended to be showed to the user asynchronously, independent of which tab is opened. Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com> --- docs/API/notifications.md | 36 ++++++++++++++++++++++++++ src/wok/control/notifications.py | 37 ++++++++++++++++++++++++++ src/wok/i18n.py | 3 +++ src/wok/model/notifications.py | 56 ++++++++++++++++++++++++++++++++++++++++ src/wok/objectstore.py | 36 ++++++++++++++++++++++++++ src/wok/server.py | 4 +++ 6 files changed, 172 insertions(+) create mode 100644 docs/API/notifications.md create mode 100644 src/wok/control/notifications.py create mode 100644 src/wok/model/notifications.py Changes in V3: - applied code review suggestions diff --git a/docs/API/notifications.md b/docs/API/notifications.md new file mode 100644 index 0000000..85e8463 --- /dev/null +++ b/docs/API/notifications.md @@ -0,0 +1,36 @@ +## REST API Specification for Notifications + +### Collection: Notifications + +**URI:** /notifications + +**Methods:** + +* **GET**: Retrieve a summarized list of current Notifications + +#### Examples +GET /notifications +[{Notification1}, {Notification2}, ...] + +### Resource: Notification + +**URI:** /notifications/*:id* + +A notification represents an asynchronous warning message sent to the web UI. + +**Methods:** + +* **GET**: Retrieve the full description of the Notification + * code: message ID + * message: message text already translated + * timestamp: first time notification was emitted + +* **DELETE**: Delete the Notification + +#### Examples +GET /notifications/KCHLIBVIRT0001W +{ + code: "KCHLIBVIRT0001W", + message: "KCHLIBVIRT0001W: Lack of storage space in guest vm-1", + timestamp: first time notification was emitted +} diff --git a/src/wok/control/notifications.py b/src/wok/control/notifications.py new file mode 100644 index 0000000..37d45f2 --- /dev/null +++ b/src/wok/control/notifications.py @@ -0,0 +1,37 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from wok.control.base import Collection, Resource +from wok.control.utils import UrlSubNode + + +@UrlSubNode('notifications', True) +class Notifications(Collection): + def __init__(self, model): + super(Notifications, self).__init__(model) + self.resource = Notification + + +class Notification(Resource): + def __init__(self, model, id): + super(Notification, self).__init__(model, id) + + @property + def data(self): + return self.info diff --git a/src/wok/i18n.py b/src/wok/i18n.py index e6087f4..5a2876f 100644 --- a/src/wok/i18n.py +++ b/src/wok/i18n.py @@ -45,6 +45,9 @@ messages = { "WOKLOG0001E": _("Invalid filter parameter. Filter parameters allowed: %(filters)s"), "WOKLOG0002E": _("Creation of log file failed: %(err)s"), + "WOKNOT0001E": _("Unable to find notification %(id)s"), + "WOKNOT0002E": _("Unable to delete notification %(id)s: %(message)s"), + "WOKOBJST0001E": _("Unable to find %(item)s in datastore"), "WOKUTILS0001E": _("Unable to reach %(url)s. Make sure it is accessible and try again."), diff --git a/src/wok/model/notifications.py b/src/wok/model/notifications.py new file mode 100644 index 0000000..77184db --- /dev/null +++ b/src/wok/model/notifications.py @@ -0,0 +1,56 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from wok.exception import NotFoundError, OperationFailed +from wok.message import WokMessage + + +class NotificationsModel(object): + def __init__(self, **kargs): + self.objstore = kargs['objstore'] + + def get_list(self): + with self.objstore as session: + return session.get_list('notification') + + +class NotificationModel(object): + def __init__(self, **kargs): + self.objstore = kargs['objstore'] + + def lookup(self, id): + with self.objstore as session: + notification = session.get('notification', str(id)) + + # use WokMessage to translate the notification + if notification: + timestamp = notification['timestamp'] + plugin = notification.pop('_plugin_name', None) + message = WokMessage(id, notification, plugin).get_text() + return {"code": id, "message": message, "timestamp": timestamp} + + raise NotFoundError("WOKNOT0001E", {'id': str(id)}) + + def delete(self, id): + try: + with self.objstore as session: + session.delete('notification', str(id)) + except Exception as e: + raise OperationFailed("WOKNOT0002E", {'id': str(id), + 'msg': e.msg()}) diff --git a/src/wok/objectstore.py b/src/wok/objectstore.py index 59354f3..ff3796c 100644 --- a/src/wok/objectstore.py +++ b/src/wok/objectstore.py @@ -23,6 +23,8 @@ import sqlite3 import threading import traceback +from datetime import datetime + try: from collections import OrderedDict except ImportError: @@ -144,3 +146,37 @@ class ObjectStore(object): # exception again wok_log.error(traceback.format_exc()) return False + + +def add_notification(code, args={}, plugin_name=None): + if not code: + wok_log.error("Unable to add notification: invalid code '%(code)s'" % + {'code': str(code)}) + return + + try: + with ObjectStore() as session: + notification = session.get('notification', code) + except NotFoundError: + notification = None + + try: + # do not update timestamp if notification already exists + timestamp = datetime.now().isoformat() if notification is None else \ + notification['timestamp'] + args.update({"_plugin_name": plugin_name, "timestamp": timestamp}) + + with ObjectStore() as session: + session.store('notification', code, args) + except Exception as e: + wok_log.error("Unable to store notification: %s" % e.message) + + +def clean_notifications(): + try: + with ObjectStore() as session: + notifications = session.get_list('notification') + for item in notifications: + session.delete('notification', item) + except Exception as e: + wok_log.error("Unable to clean notifications: %s" % e.message) diff --git a/src/wok/server.py b/src/wok/server.py index 902d4bf..a329ed4 100644 --- a/src/wok/server.py +++ b/src/wok/server.py @@ -33,6 +33,7 @@ from wok.config import config as configParser from wok.config import paths, PluginConfig, WokConfig from wok.control import sub_nodes from wok.model import model +from wok.objectstore import clean_notifications from wok.proxy import start_proxy, terminate_proxy from wok.reqlogger import RequestLogger from wok.root import WokRoot @@ -106,6 +107,9 @@ class Server(object): if dev_env: cherrypy.log.screen = True + # clean object store notifications + clean_notifications() + # close standard file handlers because we are going to use a # watchedfiled handler, otherwise we will have two file handlers # pointing to the same file, duplicating log enries -- 1.9.1

2 1

[PATCH] [Wok] Updating Makefile.am to pick fedora spec for the KVMIBM envirnment
by jkatta＠linux.vnet.ibm.com 04 Apr '16

04 Apr '16

From: Jayavardhan Katta <jkatta(a)linux.vnet.ibm.com> Signed-off-by: Jayavardhan Katta <jkatta(a)linux.vnet.ibm.com> --- Makefile.am | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.am b/Makefile.am index 86d83a0..325d0c9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -106,6 +106,8 @@ deb: contrib/make-deb.sh wok.spec: contrib/wok.spec.fedora contrib/wok.spec.suse @if test -e /etc/redhat-release; then \ ln -sf contrib/wok.spec.fedora $@ ; \ + elif test -e /etc/system-release; then \ + ln -sf contrib/wok.spec.fedora $@ ; \ elif test -e /etc/SuSE-release; then \ ln -sf contrib/wok.spec.suse $@ ; \ else \ -- 2.4.3

3 2

Patch "Implement WokMessage class" broke Ginger, Ginger-base and Gingers390x unit tests
by Daniel Henrique Barboza 04 Apr '16

04 Apr '16

The following WoK patch: "Implement WokMessage class" Broke one test case of Ginger 'test_create_existing_user_fails' with the error message below. With a similar error message, It also broke two unit tests in Gingerbase and 42 tests in Gingers390x. ====================================================================== ERROR: test_create_existing_user_fails (test_libuser.UserAdmTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/danielhb/kimchi/wok_ginger/src/wok/plugins/ginger/tests/test_libuser.py", line 68, in test_create_existing_user_fails users.create_user(user, passwd) File "/home/danielhb/kimchi/wok_ginger/src/wok/plugins/ginger/model/users.py", line 157, in create_user raise OperationFailed('GINUSER0008E', {'user': name}) File "/home/danielhb/kimchi/wok_ginger/src/wok/exception.py", line 30, in __init__ msg = WokMessage(code, args).get_text() File "/home/danielhb/kimchi/wok_ginger/src/wok/message.py", line 79, in get_text msg = self._get_translation() File "/home/danielhb/kimchi/wok_ginger/src/wok/message.py", line 48, in _get_translation wok_app = cherrypy.tree.apps[''] KeyError: '' Unless it is strictly necessary, I would rather not change the code in all those cases due to this WoK patch and WoK handle this error scenario by itself. If plug-in changes are required, please advise so I can fix them as fast as possible. Regards, Daniel

2 1

[PATCH V2] [Wok] Implement Asynchronous Notifications backend
by Lucio Correia 01 Apr '16

01 Apr '16

* There is no POST method: a notification is added through add_notification() method. * Notifications are always stored in Wok object store under 'notification' type, even if added by a plugin. * Every time Wok is started, all notifications are erased, since this is a UI feature, intended to be showed to the user asynchronously, independent of which tab is opened. Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com> --- docs/API/notifications.md | 34 ++++++++++++++++++++++++ src/wok/control/notifications.py | 37 ++++++++++++++++++++++++++ src/wok/i18n.py | 3 +++ src/wok/model/notifications.py | 56 ++++++++++++++++++++++++++++++++++++++++ src/wok/objectstore.py | 39 ++++++++++++++++++++++++++++ src/wok/server.py | 4 +++ 6 files changed, 173 insertions(+) create mode 100644 docs/API/notifications.md create mode 100644 src/wok/control/notifications.py create mode 100644 src/wok/model/notifications.py Changes in V2: - added timestamp - code as required parameter in add_notification - added DELETE api for notification This patch depends on WokMessage V3 patch diff --git a/docs/API/notifications.md b/docs/API/notifications.md new file mode 100644 index 0000000..851352f --- /dev/null +++ b/docs/API/notifications.md @@ -0,0 +1,34 @@ +## REST API Specification for Notifications + +### Collection: Notifications + +**URI:** /notifications + +**Methods:** + +* **GET**: Retrieve a summarized list of current Notifications + +#### Examples +GET /notifications +[{Notification1}, {Notification2}, ...] + +### Resource: Notification + +**URI:** /notifications/*:id* + +A task represents an asynchronous operation that is being performed by the +server. + +**Methods:** + +* **GET**: Retrieve the full description of the Notification + * code: message ID + * message: message text already translated + +#### Examples +GET /notifications/KCHLIBVIRT0001W +{ + id: "KCHLIBVIRT0001W", + message: "KCHLIBVIRT0001W: Lack of storage space in guest vm-1", + timestamp: first time notification was emitted +} diff --git a/src/wok/control/notifications.py b/src/wok/control/notifications.py new file mode 100644 index 0000000..37d45f2 --- /dev/null +++ b/src/wok/control/notifications.py @@ -0,0 +1,37 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from wok.control.base import Collection, Resource +from wok.control.utils import UrlSubNode + + +@UrlSubNode('notifications', True) +class Notifications(Collection): + def __init__(self, model): + super(Notifications, self).__init__(model) + self.resource = Notification + + +class Notification(Resource): + def __init__(self, model, id): + super(Notification, self).__init__(model, id) + + @property + def data(self): + return self.info diff --git a/src/wok/i18n.py b/src/wok/i18n.py index e6087f4..5a2876f 100644 --- a/src/wok/i18n.py +++ b/src/wok/i18n.py @@ -45,6 +45,9 @@ messages = { "WOKLOG0001E": _("Invalid filter parameter. Filter parameters allowed: %(filters)s"), "WOKLOG0002E": _("Creation of log file failed: %(err)s"), + "WOKNOT0001E": _("Unable to find notification %(id)s"), + "WOKNOT0002E": _("Unable to delete notification %(id)s: %(message)s"), + "WOKOBJST0001E": _("Unable to find %(item)s in datastore"), "WOKUTILS0001E": _("Unable to reach %(url)s. Make sure it is accessible and try again."), diff --git a/src/wok/model/notifications.py b/src/wok/model/notifications.py new file mode 100644 index 0000000..77184db --- /dev/null +++ b/src/wok/model/notifications.py @@ -0,0 +1,56 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from wok.exception import NotFoundError, OperationFailed +from wok.message import WokMessage + + +class NotificationsModel(object): + def __init__(self, **kargs): + self.objstore = kargs['objstore'] + + def get_list(self): + with self.objstore as session: + return session.get_list('notification') + + +class NotificationModel(object): + def __init__(self, **kargs): + self.objstore = kargs['objstore'] + + def lookup(self, id): + with self.objstore as session: + notification = session.get('notification', str(id)) + + # use WokMessage to translate the notification + if notification: + timestamp = notification['timestamp'] + plugin = notification.pop('_plugin_name', None) + message = WokMessage(id, notification, plugin).get_text() + return {"code": id, "message": message, "timestamp": timestamp} + + raise NotFoundError("WOKNOT0001E", {'id': str(id)}) + + def delete(self, id): + try: + with self.objstore as session: + session.delete('notification', str(id)) + except Exception as e: + raise OperationFailed("WOKNOT0002E", {'id': str(id), + 'msg': e.msg()}) diff --git a/src/wok/objectstore.py b/src/wok/objectstore.py index 59354f3..dbc3ee3 100644 --- a/src/wok/objectstore.py +++ b/src/wok/objectstore.py @@ -23,6 +23,8 @@ import sqlite3 import threading import traceback +from datetime import datetime + try: from collections import OrderedDict except ImportError: @@ -144,3 +146,40 @@ class ObjectStore(object): # exception again wok_log.error(traceback.format_exc()) return False + + +def add_notification(code, args={}, plugin_name=None): + if not code: + wok_log.error("Unable to add notification: invalide code '%(code)s'" % + {'code': str(code)}) + return + + found = True + try: + with ObjectStore() as session: + notification = session.get('notification', code) + except NotFoundError: + found = False + + try: + args.update({"_plugin_name": plugin_name, + "timestamp": datetime.now().isoformat()}) + + # do not update timestamp if notification already exists + if found: + args["timestamp"] = notification["timestamp"] + + with ObjectStore() as session: + session.store('notification', code, args) + except Exception as e: + wok_log.error("Unable to store notification: %s" % e.message) + + +def clean_notifications(): + try: + with ObjectStore() as session: + notifications = session.get_list('notification') + for item in notifications: + session.delete('notification', item) + except Exception as e: + wok_log.error("Unable to clean notifications: %s" % e.message) diff --git a/src/wok/server.py b/src/wok/server.py index 902d4bf..a329ed4 100644 --- a/src/wok/server.py +++ b/src/wok/server.py @@ -33,6 +33,7 @@ from wok.config import config as configParser from wok.config import paths, PluginConfig, WokConfig from wok.control import sub_nodes from wok.model import model +from wok.objectstore import clean_notifications from wok.proxy import start_proxy, terminate_proxy from wok.reqlogger import RequestLogger from wok.root import WokRoot @@ -106,6 +107,9 @@ class Server(object): if dev_env: cherrypy.log.screen = True + # clean object store notifications + clean_notifications() + # close standard file handlers because we are going to use a # watchedfiled handler, otherwise we will have two file handlers # pointing to the same file, duplicating log enries -- 1.9.1

3 4

[PATCH] [Wok] Remove unnecessary IDs from navbar
by peterpnns＠gmail.com 01 Apr '16

01 Apr '16

From: peterpennings <peterpnns(a)gmail.com> This patch removes unnecessary IDs from navbar to avoid warnings in HTML peterpennings (1): Remove unnecessary IDs from navbar ui/js/src/wok.main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- 2.5.0

3 3

[PATCH V3] [Wok] Implement WokMessage class
by Lucio Correia 01 Apr '16

01 Apr '16

WokMessage is a generic message implementation, which looks up and translates error messages for all plugins and Wok itself. Lookup and translation code was adapted from WokException class, which now just uses WokMessage. WokMessage is intended to be used by other modules dealing with messages, such as User Request Log and future feature Asynchronous Notifications. Signed-off-by: Lucio Correia <luciojhc(a)linux.vnet.ibm.com> --- src/wok/exception.py | 45 +++-------------------------- src/wok/message.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 41 deletions(-) create mode 100644 src/wok/message.py Changes in V3: - Aline's review diff --git a/src/wok/exception.py b/src/wok/exception.py index 023334b..52f007e 100644 --- a/src/wok/exception.py +++ b/src/wok/exception.py @@ -20,53 +20,16 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import cherrypy -import gettext -from wok.i18n import messages as _messages -from wok.template import get_lang, validate_language +from wok.message import WokMessage class WokException(Exception): def __init__(self, code='', args={}): self.code = code - - for key, value in args.iteritems(): - if isinstance(value, unicode): - continue - - # value is not unicode: convert it - try: - # In case the value formats itself to an ascii string. - args[key] = unicode(str(value), 'utf-8') - except UnicodeEncodeError: - # In case the value is a WokException or it formats - # itself to a unicode string. - args[key] = unicode(value) - - # First, check if it is a Wok error message, then search in plugin - # error messages list - msg = _messages.get(code, code) - if (msg == code) and (cherrypy.request.app): - msg = self._get_translation() - - msg = unicode(msg, 'utf-8') % args - pattern = "%s: %s" % (code, msg) - cherrypy.log.error_log.error(pattern) - Exception.__init__(self, pattern) - - def _get_translation(self): - lang = validate_language(get_lang()) - paths = cherrypy.request.app.root.paths - domain = cherrypy.request.app.root.domain - messages = cherrypy.request.app.root.messages - text = messages.get(self.code, self.code) - - try: - translation = gettext.translation(domain, paths.mo_dir, [lang]) - except: - translation = gettext - - return translation.gettext(text) + msg = WokMessage(code, args).get_text() + cherrypy.log.error_log.error(msg) + Exception.__init__(self, msg) class NotFoundError(WokException): diff --git a/src/wok/message.py b/src/wok/message.py new file mode 100644 index 0000000..414788c --- /dev/null +++ b/src/wok/message.py @@ -0,0 +1,81 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2015-2016 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import cherrypy +import gettext + +from wok.template import get_lang, validate_language + + +class WokMessage(object): + def __init__(self, code='', args={}, plugin=None): + # make all args unicode + for key, value in args.iteritems(): + if isinstance(value, unicode): + continue + + try: + # In case the value formats itself to an ascii string. + args[key] = unicode(str(value), 'utf-8') + except UnicodeEncodeError: + # In case the value is a WokException or it formats + # itself to a unicode string. + args[key] = unicode(value) + + self.code = code + self.args = args + self.plugin = plugin + + def _get_translation(self): + wok_app = cherrypy.tree.apps[''] + + # get app from plugin path if specified + if self.plugin: + app = cherrypy.tree.apps[self.plugin] + # if on request, try to get app from it + elif cherrypy.request.app: + app = cherrypy.request.app + # fallback: get root app (WokRoot) + else: + app = wok_app + + # fallback to Wok message in case plugins raise Wok exceptions + text = app.root.messages.get(self.code, None) + if text is None: + app = wok_app + text = app.root.messages.get(self.code, self.code) + + # do translation + domain = app.root.domain + paths = app.root.paths + lang = validate_language(get_lang()) + + try: + translation = gettext.translation(domain, paths.mo_dir, [lang]) + except: + translation = gettext + + return translation.gettext(text) + + def get_text(self): + msg = self._get_translation() + msg = unicode(msg, 'utf-8') % self.args + return "%s: %s" % (self.code, msg) -- 1.9.1

2 1