[Kimchi-devel] [PATCH] [WoK 1/2] /config/plugins API: backend changes

From: Daniel Henrique Barboza <danielhb(a)linux.vnet.ibm.com&gt; This patch adds a backend for a new API called /config/plugins. The idea is to be able to retrieve the 'enable' status of WoK plug-ins and also provide a way to enable/disable them. The enable|disable operation consists on two steps: - changing the 'enable=' attribute of the [WoK] section of the plugin .conf file; - the plug-in is removed/added in the cherrypy.tree on the fly. Several changes/enhancements in the backend were made to make this possible, such as: - added the 'test' parameter in the config.py.in file to make it available for reading in the backend. This parameter indicates whether WoK is running in test mode; - 'load_plugin' was moved from server.py to utils.py to make it available for utils functions to load plug-ins; - a new 'depends' attribute is now being considered in the root class of each plug-in. This is an array that indicates all the plug-ins it has a dependency on. For example, Kimchi would mark self.depends = ['gingerbase'] in its root file. The absence of this attribute means that the plug-in does not have any dependency aside from WoK. Previous /plugins API were removed because it was redundant with this work. Uni tests included. Signed-off-by: Daniel Henrique Barboza <danielhb(a)linux.vnet.ibm.com&gt; --- docs/API/config.md | 32 +++++++ docs/API/plugins.md | 13 --- src/wok/config.py.in | 5 +- src/wok/control/config.py | 31 ++++++- src/wok/control/plugins.py | 29 ------ src/wok/i18n.py | 4 + src/wok/model/plugins.py | 40 ++++++-- src/wok/server.py | 56 ++--------- src/wok/utils.py | 227 +++++++++++++++++++++++++++++++++++++++++++-- tests/test_api.py | 59 ++++++++++++ tests/test_utils.py | 75 ++++++++++++++- 11 files changed, 460 insertions(+), 111 deletions(-) delete mode 100644 docs/API/plugins.md delete mode 100644 src/wok/control/plugins.py diff --git a/docs/API/config.md b/docs/API/config.md index 4ba455e..87619ac 100644 --- a/docs/API/config.md +++ b/docs/API/config.md @@ -26,3 +26,35 @@ GET /config websockets_port: 64667, version: 2.0 } + +### Collection: Plugins + +**URI:** /config/plugins + +**Methods:** + +* **GET**: Retrieve a summarized list of all UI Plugins. + +#### Examples +GET /plugins +[{'name': 'pluginA', 'enabled': True, "depends":['pluginB'], "is_dependency_of":[]}, + {'name': 'pluginB', 'enabled': False, "depends":[], "is_dependency_of":['pluginA']}] + +### Resource: Plugins + +**URI:** /config/plugins/*:name* + +Represents the current state of a given WoK plug-in. + +**Methods:** + +* **GET**: Retrieve the state of the plug-in. + * name: The name of the plug-in. + * enabled: True if the plug-in is currently enabled in WoK, False otherwise. + +* **POST**: *See Plugin Actions* + +**Actions (POST):** + +* enable: Enable the plug-in in the configuration file. +* disable: Disable the plug-in in the configuration file. diff --git a/docs/API/plugins.md b/docs/API/plugins.md deleted file mode 100644 index aaa37b5..0000000 --- a/docs/API/plugins.md +++ /dev/null @@ -1,13 +0,0 @@ -## REST API Specification for Plugins - -### Collection: Plugins - -**URI:** /plugins - -**Methods:** - -* **GET**: Retrieve a summarized list names of all UI Plugins - -#### Examples -GET /plugins -[pluginA, pluginB, pluginC] diff --git a/src/wok/config.py.in b/src/wok/config.py.in index 9573e66..0e46b17 100644 --- a/src/wok/config.py.in +++ b/src/wok/config.py.in @@ -1,7 +1,7 @@ # # Project Wok # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # Code derived from Project Kimchi # @@ -269,6 +269,7 @@ def _get_config(): config.set("server", "environment", "production") config.set('server', 'max_body_size', '4*1024*1024') config.set("server", "server_root", "") + config.set("server", "test", "true") config.add_section("authentication") config.set("authentication", "method", "pam") config.set("authentication", "ldap_server", "") @@ -278,6 +279,8 @@ def _get_config(): config.add_section("logging") config.set("logging", "log_dir", paths.log_dir) config.set("logging", "log_level", DEFAULT_LOG_LEVEL) + config.set("logging", "access_log", "") + config.set("logging", "error_log", "") config_file = os.path.join(paths.conf_dir, 'wok.conf') if os.path.exists(config_file): diff --git a/src/wok/control/config.py b/src/wok/control/config.py index 419abc0..05383c7 100644 --- a/src/wok/control/config.py +++ b/src/wok/control/config.py @@ -17,7 +17,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from wok.control.base import Resource +from wok.control.base import Collection, Resource from wok.control.utils import UrlSubNode @@ -28,15 +28,44 @@ CONFIG_REQUESTS = { } +PLUGIN_REQUESTS = { + 'POST': { + 'enable': "WOKPLUGIN0001L", + 'disable': "WOKPLUGIN0002L", + }, +} + + @UrlSubNode("config") class Config(Resource): def __init__(self, model, id=None): super(Config, self).__init__(model, id) self.uri_fmt = '/config/%s' self.admin_methods = ['POST'] + self.plugins = Plugins(self.model) self.log_map = CONFIG_REQUESTS self.reload = self.generate_action_handler('reload') @property def data(self): return self.info + + +class Plugins(Collection): + def __init__(self, model): + super(Plugins, self).__init__(model) + self.resource = Plugin + + +class Plugin(Resource): + def __init__(self, model, ident=None): + super(Plugin, self).__init__(model, ident) + self.ident = ident + self.uri_fmt = "/config/plugins/%s" + self.log_map = PLUGIN_REQUESTS + self.enable = self.generate_action_handler('enable') + self.disable = self.generate_action_handler('disable') + + @property + def data(self): + return self.info diff --git a/src/wok/control/plugins.py b/src/wok/control/plugins.py deleted file mode 100644 index 57dfa1b..0000000 --- a/src/wok/control/plugins.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Project Wok -# -# Copyright IBM Corp, 2015-2016 -# -# Code derived from Project Kimchi -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -from wok.control.base import SimpleCollection -from wok.control.utils import UrlSubNode - - -@UrlSubNode("plugins") -class Plugins(SimpleCollection): - def __init__(self, model): - super(Plugins, self).__init__(model) diff --git a/src/wok/i18n.py b/src/wok/i18n.py index 935c9c1..d44c2f6 100644 --- a/src/wok/i18n.py +++ b/src/wok/i18n.py @@ -57,6 +57,8 @@ messages = { "WOKCONFIG0001I": _("WoK is going to restart. Existing WoK connections will be closed."), + "WOKPLUGIN0001E": _("Unable to find plug-in %(name)s"), + # These messages (ending with L) are for user log purposes "WOKASYNC0001L": _("Successfully completed task '%(target_uri)s'"), "WOKASYNC0002L": _("Failed to complete task '%(target_uri)s'"), @@ -65,4 +67,6 @@ messages = { "WOKRES0001L": _("Request made on resource"), "WOKROOT0001L": _("User '%(username)s' login"), "WOKROOT0002L": _("User '%(username)s' logout"), + "WOKPLUGIN0001L": _("Enable plug-in %(ident)s."), + "WOKPLUGIN0002L": _("Disable plug-in %(ident)s."), } diff --git a/src/wok/model/plugins.py b/src/wok/model/plugins.py index 1b8ec5e..1b39e6c 100644 --- a/src/wok/model/plugins.py +++ b/src/wok/model/plugins.py @@ -1,7 +1,7 @@ # # Project Wok # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # Code derived from Project Kimchi # @@ -19,10 +19,11 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -import cherrypy -from wok.config import get_base_plugin_uri -from wok.utils import get_enabled_plugins +from wok.exception import NotFoundError +from wok.utils import get_all_affected_plugins_by_plugin +from wok.utils import get_plugin_dependencies, get_plugins, load_plugin_conf +from wok.utils import set_plugin_state class PluginsModel(object): @@ -30,7 +31,30 @@ class PluginsModel(object): pass def get_list(self): - # Will only return plugins that were loaded correctly by WOK and are - # properly configured in cherrypy - return [plugin for (plugin, config) in get_enabled_plugins() - if get_base_plugin_uri(plugin) in cherrypy.tree.apps.keys()] + return [plugin for (plugin, config) in get_plugins()] + + +class PluginModel(object): + def __init__(self, **kargs): + pass + + def lookup(self, name): + name = name.encode('utf-8') + + plugin_conf = load_plugin_conf(name) + if not plugin_conf: + raise NotFoundError("WOKPLUGIN0001E", {'name': name}) + + depends = get_plugin_dependencies(name) + is_dependency_of = get_all_affected_plugins_by_plugin(name) + + return {"name": name, "enabled": plugin_conf['wok']['enable'], + "depends": depends, "is_dependency_of": is_dependency_of} + + def enable(self, name): + name = name.encode('utf-8') + set_plugin_state(name, True) + + def disable(self, name): + name = name.encode('utf-8') + set_plugin_state(name, False) diff --git a/src/wok/server.py b/src/wok/server.py index 48f455b..9b49c1a 100644 --- a/src/wok/server.py +++ b/src/wok/server.py @@ -1,7 +1,7 @@ # # Project Wok # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # Code derived from Project Kimchi # @@ -28,14 +28,14 @@ import os from wok import auth from wok import config from wok.config import config as configParser -from wok.config import PluginConfig, WokConfig +from wok.config import WokConfig from wok.control import sub_nodes from wok.model import model from wok.proxy import check_proxy_config from wok.reqlogger import RequestLogger from wok.root import WokRoot from wok.safewatchedfilehandler import SafeWatchedFileHandler -from wok.utils import get_enabled_plugins, import_class +from wok.utils import get_enabled_plugins, load_plugin LOGGING_LEVEL = {"debug": logging.DEBUG, @@ -153,56 +153,12 @@ class Server(object): self.app = cherrypy.tree.mount(WokRoot(model_instance, dev_env), options.server_root, self.configObj) - self._load_plugins(options) + self._load_plugins() cherrypy.lib.sessions.init() - def _load_plugins(self, options): + def _load_plugins(self): for plugin_name, plugin_config in get_enabled_plugins(): - try: - plugin_class = ('plugins.%s.%s' % - (plugin_name, - plugin_name[0].upper() + plugin_name[1:])) - del plugin_config['wok'] - plugin_config.update(PluginConfig(plugin_name)) - except KeyError: - continue - - try: - plugin_app = import_class(plugin_class)(options) - except (ImportError, Exception), e: - cherrypy.log.error_log.error( - "Failed to import plugin %s, " - "error: %s" % (plugin_class, e.message) - ) - continue - - # dynamically extend plugin config with custom data, if provided - get_custom_conf = getattr(plugin_app, "get_custom_conf", None) - if get_custom_conf is not None: - plugin_config.update(get_custom_conf()) - - # dynamically add tools.wokauth.on = True to extra plugin APIs - try: - sub_nodes = import_class('plugins.%s.control.sub_nodes' % - plugin_name) - - urlSubNodes = {} - for ident, node in sub_nodes.items(): - if node.url_auth: - ident = "/%s" % ident - urlSubNodes[ident] = {'tools.wokauth.on': True} - - plugin_config.update(urlSubNodes) - - except ImportError, e: - cherrypy.log.error_log.error( - "Failed to import subnodes for plugin %s, " - "error: %s" % (plugin_class, e.message) - ) - - cherrypy.tree.mount(plugin_app, - config.get_base_plugin_uri(plugin_name), - plugin_config) + load_plugin(plugin_name, plugin_config) def start(self): # Subscribe to SignalHandler plugin diff --git a/src/wok/utils.py b/src/wok/utils.py index 9a08001..9e6bb8a 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -1,7 +1,7 @@ # # Project Wok # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # Code derived from Project Kimchi # @@ -37,9 +37,11 @@ import xml.etree.ElementTree as ET from cherrypy.lib.reprconf import Parser from datetime import datetime, timedelta from multiprocessing import Process, Queue +from optparse import Values from threading import Timer -from wok.config import paths, PluginPaths +from wok import config +from wok.config import paths, PluginConfig, PluginPaths from wok.exception import InvalidParameter, TimeoutExpired from wok.stringutils import decode_value @@ -57,13 +59,21 @@ def is_digit(value): return False -def _load_plugin_conf(name): +def get_plugin_config_file(name): plugin_conf = PluginPaths(name).conf_file if not os.path.exists(plugin_conf): cherrypy.log.error_log.error("Plugin configuration file %s" " doesn't exist." % plugin_conf) - return + return None + return plugin_conf + + +def load_plugin_conf(name): try: + plugin_conf = get_plugin_config_file(name) + if not plugin_conf: + return None + return Parser().dict_from_file(plugin_conf) except ValueError as e: cherrypy.log.error_log.error("Failed to load plugin " @@ -71,22 +81,223 @@ def _load_plugin_conf(name): (plugin_conf, e.message)) -def get_enabled_plugins(): +def get_plugins(enabled_only=False): plugin_dir = paths.plugins_dir + try: dir_contents = os.listdir(plugin_dir) except OSError: return + + test_mode = config.config.get('server', 'test').lower() == 'true' + for name in dir_contents: if os.path.isdir(os.path.join(plugin_dir, name)): - plugin_config = _load_plugin_conf(name) + if name == 'sample' and not test_mode: + continue + + plugin_config = load_plugin_conf(name) + if not plugin_config: + continue try: - if plugin_config['wok']['enable']: - yield (name, plugin_config) + if plugin_config['wok']['enable'] is None: + continue + + plugin_enabled = plugin_config['wok']['enable'] + if enabled_only and not plugin_enabled: + continue + + yield (name, plugin_config) except (TypeError, KeyError): continue +def get_enabled_plugins(): + return get_plugins(enabled_only=True) + + +def get_plugin_app_mounted_in_cherrypy(name): + plugin_uri = '/plugins/' + name + return cherrypy.tree.apps.get(plugin_uri, None) + + +def get_plugin_dependencies(name): + app = get_plugin_app_mounted_in_cherrypy(name) + if app is None or not hasattr(app.root, 'depends'): + return [] + return app.root.depends + + +def get_all_plugins_dependent_on(name): + if not cherrypy.tree.apps: + return [] + + dependencies = [] + for plugin, app in cherrypy.tree.apps.iteritems(): + if hasattr(app.root, 'depends') and name in app.root.depends: + dependencies.append(plugin.replace('/plugins/', '')) + + return dependencies + + +def get_all_affected_plugins_by_plugin(name): + dependencies = get_all_plugins_dependent_on(name) + if len(dependencies) == 0: + return [] + + all_affected_plugins = dependencies + for dep in dependencies: + all_affected_plugins += get_all_affected_plugins_by_plugin(dep) + + return all_affected_plugins + + +def disable_plugin(name): + plugin_deps = get_all_affected_plugins_by_plugin(name) + + for dep in set(plugin_deps): + update_plugin_config_file(dep, False) + update_cherrypy_mounted_tree(dep, False) + + update_plugin_config_file(name, False) + update_cherrypy_mounted_tree(name, False) + + +def enable_plugin(name): + update_plugin_config_file(name, True) + update_cherrypy_mounted_tree(name, True) + + plugin_deps = get_plugin_dependencies(name) + + for dep in set(plugin_deps): + enable_plugin(dep) + + +def set_plugin_state(name, state): + if state is False: + disable_plugin(name) + else: + enable_plugin(name) + + +def update_plugin_config_file(name, state): + plugin_conf = get_plugin_config_file(name) + if not plugin_conf: + return + + config_contents = None + + with open(plugin_conf, 'r') as f: + config_contents = f.readlines() + + wok_section_found = False + + pattern = re.compile("^\s*enable\s*=\s*") + + for i in range(0, len(config_contents)): + if config_contents[i] == '[wok]\n': + wok_section_found = True + continue + + if pattern.match(config_contents[i]) and wok_section_found: + config_contents[i] = 'enable = %s\n' % str(state) + break + + with open(plugin_conf, 'w') as f: + f.writelines(config_contents) + + +def load_plugin(plugin_name, plugin_config): + try: + plugin_class = ('plugins.%s.%s' % + (plugin_name, + plugin_name[0].upper() + plugin_name[1:])) + del plugin_config['wok'] + plugin_config.update(PluginConfig(plugin_name)) + except KeyError: + return + + try: + options = get_plugin_config_options() + plugin_app = import_class(plugin_class)(options) + except (ImportError, Exception), e: + cherrypy.log.error_log.error( + "Failed to import plugin %s, " + "error: %s" % (plugin_class, e.message) + ) + return + + # dynamically extend plugin config with custom data, if provided + get_custom_conf = getattr(plugin_app, "get_custom_conf", None) + if get_custom_conf is not None: + plugin_config.update(get_custom_conf()) + + # dynamically add tools.wokauth.on = True to extra plugin APIs + try: + sub_nodes = import_class('plugins.%s.control.sub_nodes' % + plugin_name) + + urlSubNodes = {} + for ident, node in sub_nodes.items(): + if node.url_auth: + ident = "/%s" % ident + urlSubNodes[ident] = {'tools.wokauth.on': True} + + plugin_config.update(urlSubNodes) + + except ImportError, e: + cherrypy.log.error_log.error( + "Failed to import subnodes for plugin %s, " + "error: %s" % (plugin_class, e.message) + ) + + cherrypy.tree.mount(plugin_app, + config.get_base_plugin_uri(plugin_name), + plugin_config) + + +def is_plugin_mounted_in_cherrypy(plugin_uri): + return cherrypy.tree.apps.get(plugin_uri) is not None + + +def update_cherrypy_mounted_tree(plugin, state): + plugin_uri = '/plugin/' + plugin + + if state is False and is_plugin_mounted_in_cherrypy(plugin_uri): + del cherrypy.tree.apps[plugin_uri] + + if state is True and not is_plugin_mounted_in_cherrypy(plugin_uri): + plugin_config = load_plugin_conf(plugin) + load_plugin(plugin, plugin_config) + + +def get_plugin_config_options(): + options = Values() + + options.websockets_port = config.config.getint('server', + 'websockets_port') + options.cherrypy_port = config.config.getint('server', + 'cherrypy_port') + options.proxy_port = config.config.getint('server', 'proxy_port') + options.session_timeout = config.config.getint('server', + 'session_timeout') + + options.test = config.config.get('server', 'test') + if options.test == 'None': + options.test = None + + options.environment = config.config.get('server', 'environment') + options.server_root = config.config.get('server', 'server_root') + options.max_body_size = config.config.get('server', 'max_body_size') + + options.log_dir = config.config.get('logging', 'log_dir') + options.log_level = config.config.get('logging', 'log_level') + options.access_log = config.config.get('logging', 'access_log') + options.error_log = config.config.get('logging', 'error_log') + + return options + + def get_all_tabs(): files = [os.path.join(paths.ui_dir, 'config/tab-ext.xml')] diff --git a/tests/test_api.py b/tests/test_api.py index 1430bc1..6fbee75 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -26,6 +26,8 @@ import utils from functools import partial from wok.asynctask import AsyncTask +from wok.utils import set_plugin_state +from wok.rollbackcontext import RollbackContext test_server = None model = None @@ -54,6 +56,63 @@ class APITests(unittest.TestCase): "server_root"] self.assertEquals(sorted(keys), sorted(conf.keys())) + def test_config_plugins(self): + resp = self.request('/config/plugins') + self.assertEquals(200, resp.status) + + plugins = json.loads(resp.read()) + if len(plugins) == 0: + return + + plugin_name = '' + plugin_state = '' + for p in plugins: + if p.get('name') == 'sample': + plugin_name = p.get('name').encode('utf-8') + plugin_state = p.get('enabled') + break + else: + return + + with RollbackContext() as rollback: + rollback.prependDefer(set_plugin_state, plugin_name, + plugin_state) + + resp = self.request('/config/plugins/sample') + self.assertEquals(200, resp.status) + + resp = self.request('/config/plugins/sample/enable', + '{}', 'POST') + self.assertEquals(200, resp.status) + + resp = self.request('/config/plugins') + self.assertEquals(200, resp.status) + plugins = json.loads(resp.read()) + + for p in plugins: + if p.get('name') == 'sample': + plugin_state = p.get('enabled') + break + self.assertTrue(plugin_state) + + resp = self.request('/config/plugins/sample/disable', + '{}', 'POST') + self.assertEquals(200, resp.status) + + resp = self.request('/config/plugins') + self.assertEquals(200, resp.status) + plugins = json.loads(resp.read()) + + for p in plugins: + if p.get('name') == 'sample': + plugin_state = p.get('enabled') + break + self.assertFalse(plugin_state) + + def test_plugins_api_404(self): + resp = self.request('/plugins') + self.assertEquals(404, resp.status) + def test_user_log(self): # Login and logout to make sure there there are entries in user log hdrs = {'AUTHORIZATION': '', diff --git a/tests/test_utils.py b/tests/test_utils.py index e7fd264..e63e1a2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,10 +19,14 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import mock +import os +import tempfile import unittest from wok.exception import InvalidParameter -from wok.utils import convert_data_size +from wok.rollbackcontext import RollbackContext +from wok.utils import convert_data_size, set_plugin_state class UtilsTests(unittest.TestCase): @@ -69,3 +73,72 @@ class UtilsTests(unittest.TestCase): for d in success_data: self.assertEquals(d['got'], d['want']) + + def _get_fake_config_file_content(self, enable=True): + return """\ +[a_random_section] +# a random section for testing purposes +enable = 1 + +[wok] +# Enable plugin on Wok server (values: True|False) +enable = %s + +[fakeplugin] +# Yet another comment on this config file +enable = 2 +very_interesting_option = True +""" % str(enable) + + def _get_config_file_template(self, enable=True): + return """\ +[a_random_section] +# a random section for testing purposes +enable = 1 + +[wok] +# Enable plugin on Wok server (values: True|False) +enable = %s + +[fakeplugin] +# Yet another comment on this config file +enable = 2 +very_interesting_option = True +""" % str(enable) + + def _create_fake_config_file(self): + _, tmp_file_name = tempfile.mkstemp(suffix='.conf') + + config_contents = self._get_fake_config_file_content() + with open(tmp_file_name, 'w') as f: + f.writelines(config_contents) + + return tmp_file_name + + @mock.patch('wok.utils.get_plugin_config_file') + @mock.patch('wok.utils.update_cherrypy_mounted_tree') + def test_set_plugin_state(self, mock_update_cherrypy, mock_config_file): + mock_update_cherrypy.return_value = True + + with RollbackContext() as rollback: + + config_file_name = self._create_fake_config_file() + rollback.prependDefer(os.remove, config_file_name) + + mock_config_file.return_value = config_file_name + + set_plugin_state('pluginA', False) + with open(config_file_name, 'r') as f: + updated_conf = f.read() + self.assertEqual( + updated_conf, + self._get_config_file_template(enable=False) + ) + + set_plugin_state('pluginA', True) + with open(config_file_name, 'r') as f: + updated_conf = f.read() + self.assertEqual( + updated_conf, + self._get_config_file_template(enable=True) + ) -- 2.9.3