[PATCH] [Wok] Isolate string utils in order to avoid cyclic import
These functions are basic ones but were not able to be imported from message.py since other functions in utils were causing cyclic imports. Isolate it to avoid that problem. Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/control/base.py | 3 +- src/wok/reqlogger.py | 3 +- src/wok/stringutils.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ src/wok/utils.py | 136 +----------------------------------------- 4 files changed, 159 insertions(+), 137 deletions(-) create mode 100644 src/wok/stringutils.py IMPORTANT: apply this along with ginger and gingerbase patches. diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 69541b1..6dfc977 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -33,7 +33,8 @@ from wok.control.utils import validate_params from wok.exception import InvalidOperation, UnauthorizedError, WokException from wok.message import WokMessage from wok.reqlogger import RequestRecord -from wok.utils import get_plugin_from_request, utf8_dict, wok_log, encode_value +from wok.stringutils import encode_value, utf8_dict +from wok.utils import get_plugin_from_request, wok_log # Default request log messages diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index 8fadbcf..fd02382 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -30,7 +30,8 @@ from tempfile import NamedTemporaryFile from wok.config import config, get_log_download_path from wok.exception import InvalidParameter, OperationFailed -from wok.utils import ascii_dict, remove_old_files +from wok.stringutils import ascii_dict +from wok.utils import remove_old_files # Log search setup diff --git a/src/wok/stringutils.py b/src/wok/stringutils.py new file mode 100644 index 0000000..8f0160b --- /dev/null +++ b/src/wok/stringutils.py @@ -0,0 +1,154 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import copy +import locale + + +def ascii_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = str(value.decode('utf-8')) + + return result + + +def utf8_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = value.encode('utf-8') + + return result + + +def encode_value(val): + """ + Convert the value to string. + If its unicode, use encode otherwise str. + """ + if isinstance(val, unicode): + return val.encode('utf-8') + return str(val) + + +def decode_value(val): + """ + Converts value to unicode, + if its not an instance of unicode. + For doing so convert the val to string, + if its not instance of basestring. + """ + if not isinstance(val, basestring): + val = str(val) + if not isinstance(val, unicode): + val = val.decode('utf-8') + return val + + +def formatMeasurement(number, settings): + ''' + Refer to "Units of information" ( + http://en.wikipedia.org/wiki/Units_of_information + ) for more information about measurement units. + + @param number The number to be normalized. + @param settings + base Measurement base, accepts 2 or 10. defaults to 2. + unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. + fixed The number of digits after the decimal point. + locale The locale for formating the number if not passed + format is done as per current locale. + @returns [object] + v The number part of the measurement. + s The suffix part of the measurement including multiple and unit. + e.g., kB/s means 1000B/s, KiB/s for 1024B/s. + ''' + unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, + {"us": 'Mi', "v": 1048576}, + {"us": 'Gi', "v": 1073741824}, + {"us": 'Ti', "v": 1099511627776}, + {"us": 'Pi', "v": 1125899906842624}], + 10: [{"us": 'k', "v": 1000}, + {"us": 'M', "v": 1000000}, + {"us": 'G', "v": 1000000000}, + {"us": 'T', "v": 1000000000000}, + {"us": 'P', "v": 1000000000000000}]} + + if(not number): + return number + settings = settings or {} + unit = settings['unit'] if 'unit' in settings else 'B' + base = settings['base'] if 'base' in settings else 2 + + new_locale = settings['locale'] if 'locale' in settings else '' + + if(base != 2 and base != 10): + return encode_value(number) + unit + + fixed = settings['fixed'] + + unitMapping = unitBaseMapping[base] + for mapping in reversed(unitMapping): + suffix = mapping['us'] + startingValue = mapping['v'] + if(number < startingValue): + continue + + formatted = float(number) / startingValue + formatted = formatNumber(formatted, fixed, new_locale) + return formatted + suffix + unit + + formatted_number = formatNumber(number, fixed, new_locale) + return formatted_number+unit + + +def formatNumber(number, fixed, format_locale): + ''' + Format the number based on format_locale passed. + ''' + + # get the current locale + current_locale = locale.getlocale() + new_locale = '' + # set passed locale and set new_locale to same value. + if format_locale: + new_locale = locale.setlocale(locale.LC_ALL, format_locale) + + # Based on type of number use the correct formatter + if isinstance(number, float): + if fixed: + formatted = locale.format('%' + '.%df' % fixed, number, True) + else: + formatted = locale.format('%f', number, True) + if isinstance(number, int): + formatted = locale.format('%d', number, True) + # After formatting is done as per locale, reset the locale if changed. + if (new_locale and not current_locale[0] and not current_locale[1]): + locale.setlocale(locale.LC_ALL, 'C') + elif (new_locale): + locale.setlocale(locale.LC_ALL, current_locale[0] + "." + + current_locale[1]) + + return formatted diff --git a/src/wok/utils.py b/src/wok/utils.py index c78a77a..d6bdf0a 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -21,7 +21,6 @@ # import cherrypy -import copy import glob import grp import os @@ -33,7 +32,6 @@ import subprocess import sys import traceback import xml.etree.ElementTree as ET -import locale from cherrypy.lib.reprconf import Parser from datetime import datetime, timedelta @@ -43,6 +41,7 @@ from threading import Timer from wok.asynctask import AsyncTask from wok.config import paths, PluginPaths from wok.exception import InvalidParameter, TimeoutExpired +from wok.stringutils import decode_value wok_log = cherrypy.log.error_log @@ -137,28 +136,6 @@ def get_plugin_from_request(): return 'wok' -def ascii_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = str(value.decode('utf-8')) - - return result - - -def utf8_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = value.encode('utf-8') - - return result - - def import_class(class_path): module_name, class_name = class_path.rsplit('.', 1) try: @@ -610,114 +587,3 @@ def upgrade_objectstore_schema(objstore=None, field=None): wok_log.error("Cannot upgrade objectstore schema: %s" % e.args[0]) return False return True - - -def encode_value(val): - """ - Convert the value to string. - If its unicode, use encode otherwise str. - """ - if isinstance(val, unicode): - return val.encode('utf-8') - return str(val) - - -def decode_value(val): - """ - Converts value to unicode, - if its not an instance of unicode. - For doing so convert the val to string, - if its not instance of basestring. - """ - if not isinstance(val, basestring): - val = str(val) - if not isinstance(val, unicode): - val = val.decode('utf-8') - return val - - -def formatMeasurement(number, settings): - ''' - Refer to "Units of information" ( - http://en.wikipedia.org/wiki/Units_of_information - ) for more information about measurement units. - - @param number The number to be normalized. - @param settings - base Measurement base, accepts 2 or 10. defaults to 2. - unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. - fixed The number of digits after the decimal point. - locale The locale for formating the number if not passed - format is done as per current locale. - @returns [object] - v The number part of the measurement. - s The suffix part of the measurement including multiple and unit. - e.g., kB/s means 1000B/s, KiB/s for 1024B/s. - ''' - unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, - {"us": 'Mi', "v": 1048576}, - {"us": 'Gi', "v": 1073741824}, - {"us": 'Ti', "v": 1099511627776}, - {"us": 'Pi', "v": 1125899906842624}], - 10: [{"us": 'k', "v": 1000}, - {"us": 'M', "v": 1000000}, - {"us": 'G', "v": 1000000000}, - {"us": 'T', "v": 1000000000000}, - {"us": 'P', "v": 1000000000000000}]} - - if(not number): - return number - settings = settings or {} - unit = settings['unit'] if 'unit' in settings else 'B' - base = settings['base'] if 'base' in settings else 2 - - new_locale = settings['locale'] if 'locale' in settings else '' - - if(base != 2 and base != 10): - return encode_value(number) + unit - - fixed = settings['fixed'] - - unitMapping = unitBaseMapping[base] - for mapping in reversed(unitMapping): - suffix = mapping['us'] - startingValue = mapping['v'] - if(number < startingValue): - continue - - formatted = float(number) / startingValue - formatted = formatNumber(formatted, fixed, new_locale) - return formatted + suffix + unit - - formatted_number = formatNumber(number, fixed, new_locale) - return formatted_number+unit - - -def formatNumber(number, fixed, format_locale): - ''' - Format the number based on format_locale passed. - ''' - - # get the current locale - current_locale = locale.getlocale() - new_locale = '' - # set passed locale and set new_locale to same value. - if format_locale: - new_locale = locale.setlocale(locale.LC_ALL, format_locale) - - # Based on type of number use the correct formatter - if isinstance(number, float): - if fixed: - formatted = locale.format('%' + '.%df' % fixed, number, True) - else: - formatted = locale.format('%f', number, True) - if isinstance(number, int): - formatted = locale.format('%d', number, True) - # After formatting is done as per locale, reset the locale if changed. - if (new_locale and not current_locale[0] and not current_locale[1]): - locale.setlocale(locale.LC_ALL, 'C') - elif (new_locale): - locale.setlocale(locale.LC_ALL, current_locale[0] + "." + - current_locale[1]) - - return formatted -- 1.9.1
This patch breaks Kimchi and Ginger because you moved functions from wok.utils and those plug-ins were referencing them: Failed to import plugin plugins.kimchi.Kimchi, error: Class plugins.kimchi.Kimchi can not be imported, error: cannot import name encode_value Failed to import plugin plugins.ginger.Ginger, error: Class plugins.ginger.Ginger can not be imported, error: cannot import name encode_value The unit tests of all 3 plug-ins breaks too. On 08/04/2016 02:42 PM, Lucio Correia wrote:
These functions are basic ones but were not able to be imported from message.py since other functions in utils were causing cyclic imports. Isolate it to avoid that problem.
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/control/base.py | 3 +- src/wok/reqlogger.py | 3 +- src/wok/stringutils.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ src/wok/utils.py | 136 +----------------------------------------- 4 files changed, 159 insertions(+), 137 deletions(-) create mode 100644 src/wok/stringutils.py
IMPORTANT: apply this along with ginger and gingerbase patches.
diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 69541b1..6dfc977 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -33,7 +33,8 @@ from wok.control.utils import validate_params from wok.exception import InvalidOperation, UnauthorizedError, WokException from wok.message import WokMessage from wok.reqlogger import RequestRecord -from wok.utils import get_plugin_from_request, utf8_dict, wok_log, encode_value +from wok.stringutils import encode_value, utf8_dict +from wok.utils import get_plugin_from_request, wok_log
# Default request log messages diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index 8fadbcf..fd02382 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -30,7 +30,8 @@ from tempfile import NamedTemporaryFile
from wok.config import config, get_log_download_path from wok.exception import InvalidParameter, OperationFailed -from wok.utils import ascii_dict, remove_old_files +from wok.stringutils import ascii_dict +from wok.utils import remove_old_files
# Log search setup diff --git a/src/wok/stringutils.py b/src/wok/stringutils.py new file mode 100644 index 0000000..8f0160b --- /dev/null +++ b/src/wok/stringutils.py @@ -0,0 +1,154 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import copy +import locale + + +def ascii_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = str(value.decode('utf-8')) + + return result + + +def utf8_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = value.encode('utf-8') + + return result + + +def encode_value(val): + """ + Convert the value to string. + If its unicode, use encode otherwise str. + """ + if isinstance(val, unicode): + return val.encode('utf-8') + return str(val) + + +def decode_value(val): + """ + Converts value to unicode, + if its not an instance of unicode. + For doing so convert the val to string, + if its not instance of basestring. + """ + if not isinstance(val, basestring): + val = str(val) + if not isinstance(val, unicode): + val = val.decode('utf-8') + return val + + +def formatMeasurement(number, settings): + ''' + Refer to "Units of information" ( + http://en.wikipedia.org/wiki/Units_of_information + ) for more information about measurement units. + + @param number The number to be normalized. + @param settings + base Measurement base, accepts 2 or 10. defaults to 2. + unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. + fixed The number of digits after the decimal point. + locale The locale for formating the number if not passed + format is done as per current locale. + @returns [object] + v The number part of the measurement. + s The suffix part of the measurement including multiple and unit. + e.g., kB/s means 1000B/s, KiB/s for 1024B/s. + ''' + unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, + {"us": 'Mi', "v": 1048576}, + {"us": 'Gi', "v": 1073741824}, + {"us": 'Ti', "v": 1099511627776}, + {"us": 'Pi', "v": 1125899906842624}], + 10: [{"us": 'k', "v": 1000}, + {"us": 'M', "v": 1000000}, + {"us": 'G', "v": 1000000000}, + {"us": 'T', "v": 1000000000000}, + {"us": 'P', "v": 1000000000000000}]} + + if(not number): + return number + settings = settings or {} + unit = settings['unit'] if 'unit' in settings else 'B' + base = settings['base'] if 'base' in settings else 2 + + new_locale = settings['locale'] if 'locale' in settings else '' + + if(base != 2 and base != 10): + return encode_value(number) + unit + + fixed = settings['fixed'] + + unitMapping = unitBaseMapping[base] + for mapping in reversed(unitMapping): + suffix = mapping['us'] + startingValue = mapping['v'] + if(number < startingValue): + continue + + formatted = float(number) / startingValue + formatted = formatNumber(formatted, fixed, new_locale) + return formatted + suffix + unit + + formatted_number = formatNumber(number, fixed, new_locale) + return formatted_number+unit + + +def formatNumber(number, fixed, format_locale): + ''' + Format the number based on format_locale passed. + ''' + + # get the current locale + current_locale = locale.getlocale() + new_locale = '' + # set passed locale and set new_locale to same value. + if format_locale: + new_locale = locale.setlocale(locale.LC_ALL, format_locale) + + # Based on type of number use the correct formatter + if isinstance(number, float): + if fixed: + formatted = locale.format('%' + '.%df' % fixed, number, True) + else: + formatted = locale.format('%f', number, True) + if isinstance(number, int): + formatted = locale.format('%d', number, True) + # After formatting is done as per locale, reset the locale if changed. + if (new_locale and not current_locale[0] and not current_locale[1]): + locale.setlocale(locale.LC_ALL, 'C') + elif (new_locale): + locale.setlocale(locale.LC_ALL, current_locale[0] + "." + + current_locale[1]) + + return formatted diff --git a/src/wok/utils.py b/src/wok/utils.py index c78a77a..d6bdf0a 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -21,7 +21,6 @@ #
import cherrypy -import copy import glob import grp import os @@ -33,7 +32,6 @@ import subprocess import sys import traceback import xml.etree.ElementTree as ET -import locale
from cherrypy.lib.reprconf import Parser from datetime import datetime, timedelta @@ -43,6 +41,7 @@ from threading import Timer from wok.asynctask import AsyncTask from wok.config import paths, PluginPaths from wok.exception import InvalidParameter, TimeoutExpired +from wok.stringutils import decode_value
wok_log = cherrypy.log.error_log @@ -137,28 +136,6 @@ def get_plugin_from_request(): return 'wok'
-def ascii_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = str(value.decode('utf-8')) - - return result - - -def utf8_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = value.encode('utf-8') - - return result - - def import_class(class_path): module_name, class_name = class_path.rsplit('.', 1) try: @@ -610,114 +587,3 @@ def upgrade_objectstore_schema(objstore=None, field=None): wok_log.error("Cannot upgrade objectstore schema: %s" % e.args[0]) return False return True - - -def encode_value(val): - """ - Convert the value to string. - If its unicode, use encode otherwise str. - """ - if isinstance(val, unicode): - return val.encode('utf-8') - return str(val) - - -def decode_value(val): - """ - Converts value to unicode, - if its not an instance of unicode. - For doing so convert the val to string, - if its not instance of basestring. - """ - if not isinstance(val, basestring): - val = str(val) - if not isinstance(val, unicode): - val = val.decode('utf-8') - return val - - -def formatMeasurement(number, settings): - ''' - Refer to "Units of information" ( - http://en.wikipedia.org/wiki/Units_of_information - ) for more information about measurement units. - - @param number The number to be normalized. - @param settings - base Measurement base, accepts 2 or 10. defaults to 2. - unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. - fixed The number of digits after the decimal point. - locale The locale for formating the number if not passed - format is done as per current locale. - @returns [object] - v The number part of the measurement. - s The suffix part of the measurement including multiple and unit. - e.g., kB/s means 1000B/s, KiB/s for 1024B/s. - ''' - unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, - {"us": 'Mi', "v": 1048576}, - {"us": 'Gi', "v": 1073741824}, - {"us": 'Ti', "v": 1099511627776}, - {"us": 'Pi', "v": 1125899906842624}], - 10: [{"us": 'k', "v": 1000}, - {"us": 'M', "v": 1000000}, - {"us": 'G', "v": 1000000000}, - {"us": 'T', "v": 1000000000000}, - {"us": 'P', "v": 1000000000000000}]} - - if(not number): - return number - settings = settings or {} - unit = settings['unit'] if 'unit' in settings else 'B' - base = settings['base'] if 'base' in settings else 2 - - new_locale = settings['locale'] if 'locale' in settings else '' - - if(base != 2 and base != 10): - return encode_value(number) + unit - - fixed = settings['fixed'] - - unitMapping = unitBaseMapping[base] - for mapping in reversed(unitMapping): - suffix = mapping['us'] - startingValue = mapping['v'] - if(number < startingValue): - continue - - formatted = float(number) / startingValue - formatted = formatNumber(formatted, fixed, new_locale) - return formatted + suffix + unit - - formatted_number = formatNumber(number, fixed, new_locale) - return formatted_number+unit - - -def formatNumber(number, fixed, format_locale): - ''' - Format the number based on format_locale passed. - ''' - - # get the current locale - current_locale = locale.getlocale() - new_locale = '' - # set passed locale and set new_locale to same value. - if format_locale: - new_locale = locale.setlocale(locale.LC_ALL, format_locale) - - # Based on type of number use the correct formatter - if isinstance(number, float): - if fixed: - formatted = locale.format('%' + '.%df' % fixed, number, True) - else: - formatted = locale.format('%f', number, True) - if isinstance(number, int): - formatted = locale.format('%d', number, True) - # After formatting is done as per locale, reset the locale if changed. - if (new_locale and not current_locale[0] and not current_locale[1]): - locale.setlocale(locale.LC_ALL, 'C') - elif (new_locale): - locale.setlocale(locale.LC_ALL, current_locale[0] + "." + - current_locale[1]) - - return formatted
Just saw patches in the Ginger ML that are supposed to be applied together with this one I'll retest with all 3 patches. On 08/04/2016 03:40 PM, Daniel Henrique Barboza wrote:
This patch breaks Kimchi and Ginger because you moved functions from wok.utils and those plug-ins were referencing them:
Failed to import plugin plugins.kimchi.Kimchi, error: Class plugins.kimchi.Kimchi can not be imported, error: cannot import name encode_value Failed to import plugin plugins.ginger.Ginger, error: Class plugins.ginger.Ginger can not be imported, error: cannot import name encode_value
The unit tests of all 3 plug-ins breaks too.
On 08/04/2016 02:42 PM, Lucio Correia wrote:
These functions are basic ones but were not able to be imported from message.py since other functions in utils were causing cyclic imports. Isolate it to avoid that problem.
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/control/base.py | 3 +- src/wok/reqlogger.py | 3 +- src/wok/stringutils.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ src/wok/utils.py | 136 +----------------------------------------- 4 files changed, 159 insertions(+), 137 deletions(-) create mode 100644 src/wok/stringutils.py
IMPORTANT: apply this along with ginger and gingerbase patches.
diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 69541b1..6dfc977 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -33,7 +33,8 @@ from wok.control.utils import validate_params from wok.exception import InvalidOperation, UnauthorizedError, WokException from wok.message import WokMessage from wok.reqlogger import RequestRecord -from wok.utils import get_plugin_from_request, utf8_dict, wok_log, encode_value +from wok.stringutils import encode_value, utf8_dict +from wok.utils import get_plugin_from_request, wok_log # Default request log messages diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index 8fadbcf..fd02382 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -30,7 +30,8 @@ from tempfile import NamedTemporaryFile from wok.config import config, get_log_download_path from wok.exception import InvalidParameter, OperationFailed -from wok.utils import ascii_dict, remove_old_files +from wok.stringutils import ascii_dict +from wok.utils import remove_old_files # Log search setup diff --git a/src/wok/stringutils.py b/src/wok/stringutils.py new file mode 100644 index 0000000..8f0160b --- /dev/null +++ b/src/wok/stringutils.py @@ -0,0 +1,154 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import copy +import locale + + +def ascii_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = str(value.decode('utf-8')) + + return result + + +def utf8_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = value.encode('utf-8') + + return result + + +def encode_value(val): + """ + Convert the value to string. + If its unicode, use encode otherwise str. + """ + if isinstance(val, unicode): + return val.encode('utf-8') + return str(val) + + +def decode_value(val): + """ + Converts value to unicode, + if its not an instance of unicode. + For doing so convert the val to string, + if its not instance of basestring. + """ + if not isinstance(val, basestring): + val = str(val) + if not isinstance(val, unicode): + val = val.decode('utf-8') + return val + + +def formatMeasurement(number, settings): + ''' + Refer to "Units of information" ( + http://en.wikipedia.org/wiki/Units_of_information + ) for more information about measurement units. + + @param number The number to be normalized. + @param settings + base Measurement base, accepts 2 or 10. defaults to 2. + unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. + fixed The number of digits after the decimal point. + locale The locale for formating the number if not passed + format is done as per current locale. + @returns [object] + v The number part of the measurement. + s The suffix part of the measurement including multiple and unit. + e.g., kB/s means 1000B/s, KiB/s for 1024B/s. + ''' + unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, + {"us": 'Mi', "v": 1048576}, + {"us": 'Gi', "v": 1073741824}, + {"us": 'Ti', "v": 1099511627776}, + {"us": 'Pi', "v": 1125899906842624}], + 10: [{"us": 'k', "v": 1000}, + {"us": 'M', "v": 1000000}, + {"us": 'G', "v": 1000000000}, + {"us": 'T', "v": 1000000000000}, + {"us": 'P', "v": 1000000000000000}]} + + if(not number): + return number + settings = settings or {} + unit = settings['unit'] if 'unit' in settings else 'B' + base = settings['base'] if 'base' in settings else 2 + + new_locale = settings['locale'] if 'locale' in settings else '' + + if(base != 2 and base != 10): + return encode_value(number) + unit + + fixed = settings['fixed'] + + unitMapping = unitBaseMapping[base] + for mapping in reversed(unitMapping): + suffix = mapping['us'] + startingValue = mapping['v'] + if(number < startingValue): + continue + + formatted = float(number) / startingValue + formatted = formatNumber(formatted, fixed, new_locale) + return formatted + suffix + unit + + formatted_number = formatNumber(number, fixed, new_locale) + return formatted_number+unit + + +def formatNumber(number, fixed, format_locale): + ''' + Format the number based on format_locale passed. + ''' + + # get the current locale + current_locale = locale.getlocale() + new_locale = '' + # set passed locale and set new_locale to same value. + if format_locale: + new_locale = locale.setlocale(locale.LC_ALL, format_locale) + + # Based on type of number use the correct formatter + if isinstance(number, float): + if fixed: + formatted = locale.format('%' + '.%df' % fixed, number, True) + else: + formatted = locale.format('%f', number, True) + if isinstance(number, int): + formatted = locale.format('%d', number, True) + # After formatting is done as per locale, reset the locale if changed. + if (new_locale and not current_locale[0] and not current_locale[1]): + locale.setlocale(locale.LC_ALL, 'C') + elif (new_locale): + locale.setlocale(locale.LC_ALL, current_locale[0] + "." + + current_locale[1]) + + return formatted diff --git a/src/wok/utils.py b/src/wok/utils.py index c78a77a..d6bdf0a 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -21,7 +21,6 @@ # import cherrypy -import copy import glob import grp import os @@ -33,7 +32,6 @@ import subprocess import sys import traceback import xml.etree.ElementTree as ET -import locale from cherrypy.lib.reprconf import Parser from datetime import datetime, timedelta @@ -43,6 +41,7 @@ from threading import Timer from wok.asynctask import AsyncTask from wok.config import paths, PluginPaths from wok.exception import InvalidParameter, TimeoutExpired +from wok.stringutils import decode_value wok_log = cherrypy.log.error_log @@ -137,28 +136,6 @@ def get_plugin_from_request(): return 'wok' -def ascii_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = str(value.decode('utf-8')) - - return result - - -def utf8_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = value.encode('utf-8') - - return result - - def import_class(class_path): module_name, class_name = class_path.rsplit('.', 1) try: @@ -610,114 +587,3 @@ def upgrade_objectstore_schema(objstore=None, field=None): wok_log.error("Cannot upgrade objectstore schema: %s" % e.args[0]) return False return True - - -def encode_value(val): - """ - Convert the value to string. - If its unicode, use encode otherwise str. - """ - if isinstance(val, unicode): - return val.encode('utf-8') - return str(val) - - -def decode_value(val): - """ - Converts value to unicode, - if its not an instance of unicode. - For doing so convert the val to string, - if its not instance of basestring. - """ - if not isinstance(val, basestring): - val = str(val) - if not isinstance(val, unicode): - val = val.decode('utf-8') - return val - - -def formatMeasurement(number, settings): - ''' - Refer to "Units of information" ( - http://en.wikipedia.org/wiki/Units_of_information - ) for more information about measurement units. - - @param number The number to be normalized. - @param settings - base Measurement base, accepts 2 or 10. defaults to 2. - unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. - fixed The number of digits after the decimal point. - locale The locale for formating the number if not passed - format is done as per current locale. - @returns [object] - v The number part of the measurement. - s The suffix part of the measurement including multiple and unit. - e.g., kB/s means 1000B/s, KiB/s for 1024B/s. - ''' - unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, - {"us": 'Mi', "v": 1048576}, - {"us": 'Gi', "v": 1073741824}, - {"us": 'Ti', "v": 1099511627776}, - {"us": 'Pi', "v": 1125899906842624}], - 10: [{"us": 'k', "v": 1000}, - {"us": 'M', "v": 1000000}, - {"us": 'G', "v": 1000000000}, - {"us": 'T', "v": 1000000000000}, - {"us": 'P', "v": 1000000000000000}]} - - if(not number): - return number - settings = settings or {} - unit = settings['unit'] if 'unit' in settings else 'B' - base = settings['base'] if 'base' in settings else 2 - - new_locale = settings['locale'] if 'locale' in settings else '' - - if(base != 2 and base != 10): - return encode_value(number) + unit - - fixed = settings['fixed'] - - unitMapping = unitBaseMapping[base] - for mapping in reversed(unitMapping): - suffix = mapping['us'] - startingValue = mapping['v'] - if(number < startingValue): - continue - - formatted = float(number) / startingValue - formatted = formatNumber(formatted, fixed, new_locale) - return formatted + suffix + unit - - formatted_number = formatNumber(number, fixed, new_locale) - return formatted_number+unit - - -def formatNumber(number, fixed, format_locale): - ''' - Format the number based on format_locale passed. - ''' - - # get the current locale - current_locale = locale.getlocale() - new_locale = '' - # set passed locale and set new_locale to same value. - if format_locale: - new_locale = locale.setlocale(locale.LC_ALL, format_locale) - - # Based on type of number use the correct formatter - if isinstance(number, float): - if fixed: - formatted = locale.format('%' + '.%df' % fixed, number, True) - else: - formatted = locale.format('%f', number, True) - if isinstance(number, int): - formatted = locale.format('%d', number, True) - # After formatting is done as per locale, reset the locale if changed. - if (new_locale and not current_locale[0] and not current_locale[1]): - locale.setlocale(locale.LC_ALL, 'C') - elif (new_locale): - locale.setlocale(locale.LC_ALL, current_locale[0] + "." + - current_locale[1]) - - return formatted
Applied to master. Thanks! On 08/04/2016 02:42 PM, Lucio Correia wrote:
These functions are basic ones but were not able to be imported from message.py since other functions in utils were causing cyclic imports. Isolate it to avoid that problem.
Signed-off-by: Lucio Correia <luciojhc@linux.vnet.ibm.com> --- src/wok/control/base.py | 3 +- src/wok/reqlogger.py | 3 +- src/wok/stringutils.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ src/wok/utils.py | 136 +----------------------------------------- 4 files changed, 159 insertions(+), 137 deletions(-) create mode 100644 src/wok/stringutils.py
IMPORTANT: apply this along with ginger and gingerbase patches.
diff --git a/src/wok/control/base.py b/src/wok/control/base.py index 69541b1..6dfc977 100644 --- a/src/wok/control/base.py +++ b/src/wok/control/base.py @@ -33,7 +33,8 @@ from wok.control.utils import validate_params from wok.exception import InvalidOperation, UnauthorizedError, WokException from wok.message import WokMessage from wok.reqlogger import RequestRecord -from wok.utils import get_plugin_from_request, utf8_dict, wok_log, encode_value +from wok.stringutils import encode_value, utf8_dict +from wok.utils import get_plugin_from_request, wok_log
# Default request log messages diff --git a/src/wok/reqlogger.py b/src/wok/reqlogger.py index 8fadbcf..fd02382 100644 --- a/src/wok/reqlogger.py +++ b/src/wok/reqlogger.py @@ -30,7 +30,8 @@ from tempfile import NamedTemporaryFile
from wok.config import config, get_log_download_path from wok.exception import InvalidParameter, OperationFailed -from wok.utils import ascii_dict, remove_old_files +from wok.stringutils import ascii_dict +from wok.utils import remove_old_files
# Log search setup diff --git a/src/wok/stringutils.py b/src/wok/stringutils.py new file mode 100644 index 0000000..8f0160b --- /dev/null +++ b/src/wok/stringutils.py @@ -0,0 +1,154 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2016 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import copy +import locale + + +def ascii_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = str(value.decode('utf-8')) + + return result + + +def utf8_dict(base, overlay=None): + result = copy.deepcopy(base) + result.update(overlay or {}) + + for key, value in result.iteritems(): + if isinstance(value, unicode): + result[key] = value.encode('utf-8') + + return result + + +def encode_value(val): + """ + Convert the value to string. + If its unicode, use encode otherwise str. + """ + if isinstance(val, unicode): + return val.encode('utf-8') + return str(val) + + +def decode_value(val): + """ + Converts value to unicode, + if its not an instance of unicode. + For doing so convert the val to string, + if its not instance of basestring. + """ + if not isinstance(val, basestring): + val = str(val) + if not isinstance(val, unicode): + val = val.decode('utf-8') + return val + + +def formatMeasurement(number, settings): + ''' + Refer to "Units of information" ( + http://en.wikipedia.org/wiki/Units_of_information + ) for more information about measurement units. + + @param number The number to be normalized. + @param settings + base Measurement base, accepts 2 or 10. defaults to 2. + unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. + fixed The number of digits after the decimal point. + locale The locale for formating the number if not passed + format is done as per current locale. + @returns [object] + v The number part of the measurement. + s The suffix part of the measurement including multiple and unit. + e.g., kB/s means 1000B/s, KiB/s for 1024B/s. + ''' + unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, + {"us": 'Mi', "v": 1048576}, + {"us": 'Gi', "v": 1073741824}, + {"us": 'Ti', "v": 1099511627776}, + {"us": 'Pi', "v": 1125899906842624}], + 10: [{"us": 'k', "v": 1000}, + {"us": 'M', "v": 1000000}, + {"us": 'G', "v": 1000000000}, + {"us": 'T', "v": 1000000000000}, + {"us": 'P', "v": 1000000000000000}]} + + if(not number): + return number + settings = settings or {} + unit = settings['unit'] if 'unit' in settings else 'B' + base = settings['base'] if 'base' in settings else 2 + + new_locale = settings['locale'] if 'locale' in settings else '' + + if(base != 2 and base != 10): + return encode_value(number) + unit + + fixed = settings['fixed'] + + unitMapping = unitBaseMapping[base] + for mapping in reversed(unitMapping): + suffix = mapping['us'] + startingValue = mapping['v'] + if(number < startingValue): + continue + + formatted = float(number) / startingValue + formatted = formatNumber(formatted, fixed, new_locale) + return formatted + suffix + unit + + formatted_number = formatNumber(number, fixed, new_locale) + return formatted_number+unit + + +def formatNumber(number, fixed, format_locale): + ''' + Format the number based on format_locale passed. + ''' + + # get the current locale + current_locale = locale.getlocale() + new_locale = '' + # set passed locale and set new_locale to same value. + if format_locale: + new_locale = locale.setlocale(locale.LC_ALL, format_locale) + + # Based on type of number use the correct formatter + if isinstance(number, float): + if fixed: + formatted = locale.format('%' + '.%df' % fixed, number, True) + else: + formatted = locale.format('%f', number, True) + if isinstance(number, int): + formatted = locale.format('%d', number, True) + # After formatting is done as per locale, reset the locale if changed. + if (new_locale and not current_locale[0] and not current_locale[1]): + locale.setlocale(locale.LC_ALL, 'C') + elif (new_locale): + locale.setlocale(locale.LC_ALL, current_locale[0] + "." + + current_locale[1]) + + return formatted diff --git a/src/wok/utils.py b/src/wok/utils.py index c78a77a..d6bdf0a 100644 --- a/src/wok/utils.py +++ b/src/wok/utils.py @@ -21,7 +21,6 @@ #
import cherrypy -import copy import glob import grp import os @@ -33,7 +32,6 @@ import subprocess import sys import traceback import xml.etree.ElementTree as ET -import locale
from cherrypy.lib.reprconf import Parser from datetime import datetime, timedelta @@ -43,6 +41,7 @@ from threading import Timer from wok.asynctask import AsyncTask from wok.config import paths, PluginPaths from wok.exception import InvalidParameter, TimeoutExpired +from wok.stringutils import decode_value
wok_log = cherrypy.log.error_log @@ -137,28 +136,6 @@ def get_plugin_from_request(): return 'wok'
-def ascii_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = str(value.decode('utf-8')) - - return result - - -def utf8_dict(base, overlay=None): - result = copy.deepcopy(base) - result.update(overlay or {}) - - for key, value in result.iteritems(): - if isinstance(value, unicode): - result[key] = value.encode('utf-8') - - return result - - def import_class(class_path): module_name, class_name = class_path.rsplit('.', 1) try: @@ -610,114 +587,3 @@ def upgrade_objectstore_schema(objstore=None, field=None): wok_log.error("Cannot upgrade objectstore schema: %s" % e.args[0]) return False return True - - -def encode_value(val): - """ - Convert the value to string. - If its unicode, use encode otherwise str. - """ - if isinstance(val, unicode): - return val.encode('utf-8') - return str(val) - - -def decode_value(val): - """ - Converts value to unicode, - if its not an instance of unicode. - For doing so convert the val to string, - if its not instance of basestring. - """ - if not isinstance(val, basestring): - val = str(val) - if not isinstance(val, unicode): - val = val.decode('utf-8') - return val - - -def formatMeasurement(number, settings): - ''' - Refer to "Units of information" ( - http://en.wikipedia.org/wiki/Units_of_information - ) for more information about measurement units. - - @param number The number to be normalized. - @param settings - base Measurement base, accepts 2 or 10. defaults to 2. - unit The unit of the measurement, e.g., B, Bytes/s, bps, etc. - fixed The number of digits after the decimal point. - locale The locale for formating the number if not passed - format is done as per current locale. - @returns [object] - v The number part of the measurement. - s The suffix part of the measurement including multiple and unit. - e.g., kB/s means 1000B/s, KiB/s for 1024B/s. - ''' - unitBaseMapping = {2: [{"us": 'Ki', "v": 1024}, - {"us": 'Mi', "v": 1048576}, - {"us": 'Gi', "v": 1073741824}, - {"us": 'Ti', "v": 1099511627776}, - {"us": 'Pi', "v": 1125899906842624}], - 10: [{"us": 'k', "v": 1000}, - {"us": 'M', "v": 1000000}, - {"us": 'G', "v": 1000000000}, - {"us": 'T', "v": 1000000000000}, - {"us": 'P', "v": 1000000000000000}]} - - if(not number): - return number - settings = settings or {} - unit = settings['unit'] if 'unit' in settings else 'B' - base = settings['base'] if 'base' in settings else 2 - - new_locale = settings['locale'] if 'locale' in settings else '' - - if(base != 2 and base != 10): - return encode_value(number) + unit - - fixed = settings['fixed'] - - unitMapping = unitBaseMapping[base] - for mapping in reversed(unitMapping): - suffix = mapping['us'] - startingValue = mapping['v'] - if(number < startingValue): - continue - - formatted = float(number) / startingValue - formatted = formatNumber(formatted, fixed, new_locale) - return formatted + suffix + unit - - formatted_number = formatNumber(number, fixed, new_locale) - return formatted_number+unit - - -def formatNumber(number, fixed, format_locale): - ''' - Format the number based on format_locale passed. - ''' - - # get the current locale - current_locale = locale.getlocale() - new_locale = '' - # set passed locale and set new_locale to same value. - if format_locale: - new_locale = locale.setlocale(locale.LC_ALL, format_locale) - - # Based on type of number use the correct formatter - if isinstance(number, float): - if fixed: - formatted = locale.format('%' + '.%df' % fixed, number, True) - else: - formatted = locale.format('%f', number, True) - if isinstance(number, int): - formatted = locale.format('%d', number, True) - # After formatting is done as per locale, reset the locale if changed. - if (new_locale and not current_locale[0] and not current_locale[1]): - locale.setlocale(locale.LC_ALL, 'C') - elif (new_locale): - locale.setlocale(locale.LC_ALL, current_locale[0] + "." + - current_locale[1]) - - return formatted
participants (2)
-
Daniel Henrique Barboza -
Lucio Correia