[Kimchi-devel] [PATCH 04/17] V7 Ginger Base : base folder files part 3

From: chandrureddy <chandra(a)linux.vnet.ibm.com&gt; --- src/wok/plugins/gingerbase/gingerbase.py | 60 +++++ src/wok/plugins/gingerbase/i18n.py | 96 +++++++ src/wok/plugins/gingerbase/lscpu.py | 126 ++++++++++ src/wok/plugins/gingerbase/mockmodel.py | 220 ++++++++++++++++ src/wok/plugins/gingerbase/swupdate.py | 415 +++++++++++++++++++++++++++++++ src/wok/plugins/gingerbase/utils.py | 82 ++++++ src/wok/plugins/kimchi/swupdate.py | 412 ------------------------------ 7 files changed, 999 insertions(+), 412 deletions(-) create mode 100644 src/wok/plugins/gingerbase/gingerbase.py create mode 100644 src/wok/plugins/gingerbase/i18n.py create mode 100644 src/wok/plugins/gingerbase/lscpu.py create mode 100644 src/wok/plugins/gingerbase/mockmodel.py create mode 100644 src/wok/plugins/gingerbase/swupdate.py create mode 100644 src/wok/plugins/gingerbase/utils.py delete mode 100644 src/wok/plugins/kimchi/swupdate.py diff --git a/src/wok/plugins/gingerbase/gingerbase.py b/src/wok/plugins/gingerbase/gingerbase.py new file mode 100644 index 0000000..0d3709e --- /dev/null +++ b/src/wok/plugins/gingerbase/gingerbase.py @@ -0,0 +1,60 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import json +import os + +from wok.plugins.gingerbase import config, mockmodel +from wok.plugins.gingerbase.i18n import messages +from wok.plugins.gingerbase.control import sub_nodes +from wok.plugins.gingerbase.model import model as gingerBaseModel +from wok.root import WokRoot + + +class GingerBase(WokRoot): + def __init__(self, wok_options): + if hasattr(wok_options, "model"): + self.model = wok_options.model + elif wok_options.test: + self.model = mockmodel.MockModel() + else: + self.model = gingerBaseModel.Model() + + dev_env = wok_options.environment != 'production' + super(GingerBase, self).__init__(self.model, dev_env) + + for ident, node in sub_nodes.items(): + setattr(self, ident, node(self.model)) + + self.api_schema = json.load(open(os.path.join(os.path.dirname( + os.path.abspath(__file__)), 'API.json'))) + self.paths = config.gingerBasePaths + self.domain = 'gingerbase' + self.messages = messages + + make_dirs = [ + os.path.dirname(os.path.abspath(config.get_object_store())), + os.path.abspath(config.get_debugreports_path()) + ] + for directory in make_dirs: + if not os.path.isdir(directory): + os.makedirs(directory) + + def get_custom_conf(self): + return config.GingerBaseConfig() diff --git a/src/wok/plugins/gingerbase/i18n.py b/src/wok/plugins/gingerbase/i18n.py new file mode 100644 index 0000000..fbc2516 --- /dev/null +++ b/src/wok/plugins/gingerbase/i18n.py @@ -0,0 +1,96 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import gettext + +_ = gettext.gettext + + +messages = { + "GGBAPI0001E": _("Unknown parameter %(value)s"), + + "GGBDISKS0001E": _("Error while getting block devices. Details: %(err)s"), + "GGBDISKS0002E": _("Error while getting block device information for %(device)s."), + + "GGBDR0001E": _("Debug report %(name)s does not exist"), + "GGBDR0002E": _("Debug report tool not found in system"), + "GGBDR0003E": _("Unable to create debug report %(name)s. Details: %(err)s."), + "GGBDR0004E": _("Can not find any debug report with the given name %(name)s"), + "GGBDR0005E": _("Unable to generate debug report %(name)s. Details: %(err)s"), + "GGBDR0006E": _("You should give a name for the debug report file."), + "GGBDR0007E": _("Debug report name must be a string. Only letters, digits, underscore ('_') and " + "hyphen ('-') are allowed."), + "GGBDR0008E": _("The debug report with specified name \"%(name)s\" already exists. Please use another one."), + + "GGBPART0001E": _("Partition %(name)s does not exist in the host"), + + "GGBHOST0001E": _("Unable to shutdown host machine as there are running virtual machines"), + "GGBHOST0002E": _("Unable to reboot host machine as there are running virtual machines"), + "GGBHOST0005E": _("When specifying CPU topology, each element must be an integer greater than zero."), + + "GGBPKGUPD0001E": _("No packages marked for update"), + "GGBPKGUPD0002E": _("Package %(name)s is not marked to be updated."), + "GGBPKGUPD0003E": _("Error while getting packages marked to be updated. Details: %(err)s"), + "GGBPKGUPD0004E": _("There is no compatible package manager for this system."), + "GGBPKGUPD0005E": _("There is a package manager instance running in the system."), + + "GGBREPOS0001E": _("YUM Repository ID must be one word only string."), + "GGBREPOS0002E": _("Repository URL must be an http://, ftp:// or file:// URL."), + "GGBREPOS0003E": _("Repository configuration is a dictionary with specific values according to repository type."), + "GGBREPOS0004E": _("Distribution to DEB repository must be a string"), + "GGBREPOS0005E": _("Components to DEB repository must be listed in a array"), + "GGBREPOS0006E": _("Components to DEB repository must be a string"), + "GGBREPOS0007E": _("Mirror list to repository must be a string"), + "GGBREPOS0008E": _("YUM Repository name must be string."), + "GGBREPOS0009E": _("GPG check must be a boolean value."), + "GGBREPOS0010E": _("GPG key must be a URL pointing to the ASCII-armored file."), + "GGBREPOS0011E": _("Could not update repository %(repo_id)s."), + "GGBREPOS0012E": _("Repository %(repo_id)s does not exist."), + "GGBREPOS0013E": _("Specify repository base URL, mirror list or metalink in order to create or " + "update a YUM repository."), + "GGBREPOS0014E": _("Repository management tool was not recognized for your system."), + "GGBREPOS0015E": _("Repository %(repo_id)s is already enabled."), + "GGBREPOS0016E": _("Repository %(repo_id)s is already disabled."), + "GGBREPOS0017E": _("Could not remove repository %(repo_id)s."), + "GGBREPOS0018E": _("Could not write repository configuration file %(repo_file)s"), + "GGBREPOS0019E": _("Specify repository distribution in order to create a DEB repository."), + "GGBREPOS0020E": _("Could not enable repository %(repo_id)s."), + "GGBREPOS0021E": _("Could not disable repository %(repo_id)s."), + "GGBREPOS0022E": _("YUM Repository ID already exists"), + "GGBREPOS0023E": _("YUM Repository name must be a string"), + "GGBREPOS0024E": _("Unable to list repositories. Details: '%(err)s'"), + "GGBREPOS0025E": _("Unable to retrieve repository information. Details: '%(err)s'"), + "GGBREPOS0026E": _("Unable to add repository. Details: '%(err)s'"), + "GGBREPOS0027E": _("Unable to remove repository. Details: '%(err)s'"), + "GGBREPOS0028E": _("Configuration items: '%(items)s' are not supported by repository manager"), + "GGBREPOS0029E": _("Repository metalink must be an http://, ftp:// or file:// URL."), + "GGBREPOS0030E": _("Cannot specify mirrorlist and metalink at the same time."), + + + "GGBCPUINF0001E": _("The number of vCPUs is too large for this system."), + "GGBCPUINF0002E": _("Invalid vCPU/topology combination."), + "GGBCPUINF0003E": _("This host (or current configuration) does not allow CPU topology."), + "GGBCPUINF0004E": _("This host (or current configuration) does not allow to fetch lscpu details."), + "GGBCPUINF0005E": _("This host (or current configuration) does not provide Socket(s) information."), + "GGBCPUINF0006E": _("This host (or current configuration) does not provide Core(s) per socket information."), + "GGBCPUINF0007E": _("This host (or current configuration) does not provide Thread(s) per core information."), + +} diff --git a/src/wok/plugins/gingerbase/lscpu.py b/src/wok/plugins/gingerbase/lscpu.py new file mode 100644 index 0000000..0fc1e72 --- /dev/null +++ b/src/wok/plugins/gingerbase/lscpu.py @@ -0,0 +1,126 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import logging + +from wok.utils import run_command +from wok.exception import NotFoundError + + +class LsCpu(object): + """ + Get CPU information about a CPU hyper threading/architecture on x86 + """ + def log_error(e): + """ + param e: error details to be logged + """ + log = logging.getLogger('Util') + log.warning('Exception in fetching the CPU architecture details: %s', + e) + + def __init__(self): + self.lsCpuInfo = {} + try: + # lscpu - display information about the CPU architecture + out, error, rc = run_command(['lscpu']) + # Output of lscpu on x86 is expected to be: + # Architecture: x86_64 + # CPU op-mode(s): 32-bit, 64-bit + # Byte Order: Little Endian + # CPU(s): 4 + # On-line CPU(s) list: 0-3 + # Thread(s) per core: 2 + # Core(s) per socket: 2 + # Socket(s): 1 + # NUMA node(s): 1 + # Vendor ID: GenuineIntel + # CPU family: 6 + # Model: 42 + # Model name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz + # Stepping: 7 + # CPU MHz: 976.421 + # CPU max MHz: 3300.0000 + # CPU min MHz: 800.0000 + # BogoMIPS: 5182.99 + # Virtualization: VT-x + # L1d cache: 32K + # L1i cache: 32K + # L2 cache: 256K + # L3 cache: 3072K + # NUMA node0 CPU(s): 0-3 + + if not rc and (not out.isspace()): + lscpuout = out.split('\n') + if lscpuout and len(lscpuout) > 0: + for line in lscpuout: + if ":" in line and (len(line.split(':')) == 2): + self.lsCpuInfo[line.split(':')[0].strip()] = \ + line.split(':')[1].strip() + else: + continue + except Exception, e: + self.log_error(e) + raise NotFoundError("GGBCPUINF0004E") + + def get_sockets(self): + """ + param self: object of the class self + return: Socket(s) (information about the CPU architecture) + """ + try: + sockets = "Socket(s)" + if len(self.lsCpuInfo) > 0 and sockets in self.lsCpuInfo.keys(): + return int(self.lsCpuInfo[sockets]) + else: + raise NotFoundError("GGBCPUINF0005E") + except IndexError, e: + self.log_error(e) + raise NotFoundError("GGBCPUINF0005E") + + def get_cores_per_socket(self): + """ + param self: object of the class self + return: Core(s) per socket (information about the CPU architecture) + """ + try: + cores_per_socket = "Core(s) per socket" + if len(self.lsCpuInfo) > 0 and cores_per_socket \ + in self.lsCpuInfo.keys(): + return int(self.lsCpuInfo[cores_per_socket]) + else: + raise NotFoundError("GGBCPUINF0006E") + except IndexError, e: + self.log_error(e) + raise NotFoundError("GGBCPUINF0006E") + + def get_threads_per_core(self): + """ + param self: object of the class self + return: Thread(s) per core (information about the CPU architecture) + """ + try: + threads_per_core = "Thread(s) per core" + if len(self.lsCpuInfo) > 0 and threads_per_core \ + in self.lsCpuInfo.keys(): + return int(self.lsCpuInfo[threads_per_core]) + else: + raise NotFoundError("GGBCPUINF0007E") + except IndexError, e: + self.log_error(e) + raise NotFoundError("GGBCPUINF0007E") diff --git a/src/wok/plugins/gingerbase/mockmodel.py b/src/wok/plugins/gingerbase/mockmodel.py new file mode 100644 index 0000000..0b5ae22 --- /dev/null +++ b/src/wok/plugins/gingerbase/mockmodel.py @@ -0,0 +1,220 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import lxml.etree as ET +import os +import random +import time + +from wok.objectstore import ObjectStore +from wok.utils import add_task, wok_log + +from wok.plugins.gingerbase import config +from wok.plugins.gingerbase.model import cpuinfo +from wok.plugins.gingerbase.model.debugreports import DebugReportsModel +from wok.plugins.gingerbase.model.model import Model + +fake_user = {'root': 'letmein!'} +mockmodel_defaults = {'domain': 'test', 'arch': 'i686'} + + +class MockModel(Model): + + def __init__(self, objstore_loc=None): + # Override osinfo.defaults to ajust the values according to + # test:///default driver + + self._mock_partitions = MockPartitions() + self._mock_swupdate = MockSoftwareUpdate() + self._mock_repositories = MockRepositories() + + cpuinfo.get_topo_capabilities = \ + MockModel.get_topo_capabilities + + super(MockModel, self).__init__(objstore_loc) + self.objstore_loc = objstore_loc + self.objstore = ObjectStore(objstore_loc) + + # The MockModel methods are instantiated on runtime according to Model + # and BaseModel + # Because that a normal method override will not work here + # Instead of that we also need to do the override on runtime + for method in dir(self): + if method.startswith('_mock_'): + mock_method = getattr(self, method) + if not callable(mock_method): + continue + + m = method[6:] + model_method = getattr(self, m) + setattr(self, '_model_' + m, model_method) + setattr(self, m, mock_method) + + DebugReportsModel._gen_debugreport_file = self._gen_debugreport_file + + def reset(self): + self._mock_swupdate = MockSoftwareUpdate() + self._mock_repositories = MockRepositories() + + if hasattr(self, 'objstore'): + self.objstore = ObjectStore(self.objstore_loc) + + @staticmethod + def get_topo_capabilities(conn): + # The libvirt test driver doesn't return topology. + xml = "<topology sockets='1' cores='2' threads='2'/>" + return ET.fromstring(xml) + + def _gen_debugreport_file(self, name): + return add_task('/plugins/gingerbase/debugreports/%s' % name, + self._create_log, self.objstore, name) + + def _create_log(self, cb, name): + path = config.get_debugreports_path() + tmpf = os.path.join(path, name + '.tmp') + realf = os.path.join(path, name + '.txt') + length = random.randint(1000, 10000) + with open(tmpf, 'w') as fd: + while length: + fd.write('I am logged') + length = length - 1 + os.rename(tmpf, realf) + cb("OK", True) + + def _mock_host_shutdown(self, *name): + wok_log.info("The host system will be shutted down") + + def _mock_host_reboot(self, *name): + wok_log.info("The host system will be rebooted") + + def _mock_partitions_get_list(self): + return self._mock_partitions.partitions.keys() + + def _mock_partition_lookup(self, name): + return self._mock_partitions.partitions[name] + + def _mock_packagesupdate_get_list(self): + return self._mock_swupdate.pkgs.keys() + + def _mock_packageupdate_lookup(self, pkg_name): + return self._mock_swupdate.pkgs[pkg_name] + + def _mock_host_swupdate(self, args=None): + task_id = add_task('/plugins/gingerbase/host/swupdate', + self._mock_swupdate.doUpdate, + self.objstore) + return self.task_lookup(task_id) + + def _mock_repositories_get_list(self): + return self._mock_repositories.repos.keys() + + def _mock_repositories_create(self, params): + # Create a repo_id if not given by user. The repo_id will follow + # the format gingerbase_repo_<integer>, where integer is the number of + # seconds since the Epoch (January 1st, 1970), in UTC. + repo_id = params.get('repo_id', None) + if repo_id is None: + repo_id = "gingerbase_repo_%s" % str(int(time.time() * 1000)) + params.update({'repo_id': repo_id}) + + config = params.get('config', {}) + info = {'repo_id': repo_id, + 'baseurl': params['baseurl'], + 'enabled': True, + 'config': {'repo_name': config.get('repo_name', repo_id), + 'gpgkey': config.get('gpgkey', []), + 'gpgcheck': True, + 'mirrorlist': params.get('mirrorlist', '')}} + self._mock_repositories.repos[repo_id] = info + return repo_id + + def _mock_repository_lookup(self, repo_id): + return self._mock_repositories.repos[repo_id] + + def _mock_repository_delete(self, repo_id): + del self._mock_repositories.repos[repo_id] + + def _mock_repository_enable(self, repo_id): + self._mock_repositories.repos[repo_id]['enabled'] = True + + def _mock_repository_disable(self, repo_id): + self._mock_repositories.repos[repo_id]['enabled'] = False + + def _mock_repository_update(self, repo_id, params): + self._mock_repositories.repos[repo_id].update(params) + return repo_id + + +class MockPartitions(object): + def __init__(self): + self.partitions = {"vdx": {"available": True, "name": "vdx", + "fstype": "", "path": "/dev/vdx", + "mountpoint": "", "type": "disk", + "size": "2147483648"}, + "vdz": {"available": True, "name": "vdz", + "fstype": "", "path": "/dev/vdz", + "mountpoint": "", "type": "disk", + "size": "2147483648"}} + + +class MockSoftwareUpdate(object): + def __init__(self): + self.pkgs = { + 'udevmountd': {'repository': 'openSUSE-13.1-Update', + 'version': '0.81.5-14.1', + 'arch': 'x86_64', + 'package_name': 'udevmountd'}, + 'sysconfig-network': {'repository': 'openSUSE-13.1-Extras', + 'version': '0.81.5-14.1', + 'arch': 'x86_64', + 'package_name': 'sysconfig-network'}, + 'libzypp': {'repository': 'openSUSE-13.1-Update', + 'version': '13.9.0-10.1', + 'arch': 'noarch', + 'package_name': 'libzypp'}} + self._num2update = 3 + + def doUpdate(self, cb, params): + msgs = [] + for pkg in self.pkgs.keys(): + msgs.append("Updating package %s" % pkg) + cb('\n'.join(msgs)) + time.sleep(1) + + time.sleep(2) + msgs.append("All packages updated") + cb('\n'.join(msgs), True) + + # After updating all packages any package should be listed to be + # updated, so reset self._packages + self.pkgs = {} + + +class MockRepositories(object): + def __init__(self): + self.repos = {"gingerbase_repo_1392167832": + {"repo_id": "gingerbase_repo_1392167832", + "enabled": True, + "baseurl": "http://www.fedora.org", + "config": {"repo_name": "gingerbase_repo_1392167832", + "gpgkey": [], + "gpgcheck": True, + "mirrorlist": ""}}} diff --git a/src/wok/plugins/gingerbase/swupdate.py b/src/wok/plugins/gingerbase/swupdate.py new file mode 100644 index 0000000..6ac98e2 --- /dev/null +++ b/src/wok/plugins/gingerbase/swupdate.py @@ -0,0 +1,415 @@ +# +# Project Ginger Base +# +# Copyright IBM, Corp. 2014-2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import fcntl +import os +import signal +import subprocess +import time +from configobj import ConfigObj, ConfigObjError +from psutil import pid_exists + +from wok.basemodel import Singleton +from wok.exception import NotFoundError, OperationFailed +from wok.utils import run_command, wok_log + +from wok.plugins.gingerbase.config import gingerBaseLock +from wok.plugins.gingerbase.yumparser import get_yum_packages_list_update + + +class SoftwareUpdate(object): + __metaclass__ = Singleton + + """ + Class to represent and operate with OS software update. + """ + def __init__(self): + # This stores all packages to be updated for Ginger Base perspective. + # It's a dictionary of dictionaries, in the format + # {'package_name': package}, + # where: + # package = {'package_name': <string>, 'version': <string>, + # 'arch': <string>, 'repository': <string> + # } + self._packages = {} + + # This stores the number of packages to update + self._num2update = 0 + + # Get the distro of host machine and creates an object related to + # correct package management system + try: + __import__('yum') + wok_log.info("Loading YumUpdate features.") + self._pkg_mnger = YumUpdate() + except ImportError: + try: + __import__('apt') + wok_log.info("Loading AptUpdate features.") + self._pkg_mnger = AptUpdate() + except ImportError: + zypper_help = ["zypper", "--help"] + (stdout, stderr, returncode) = run_command(zypper_help) + if returncode == 0: + wok_log.info("Loading ZypperUpdate features.") + self._pkg_mnger = ZypperUpdate() + else: + raise Exception("There is no compatible package manager " + "for this system.") + + def _scanUpdates(self): + """ + Update self._packages with packages to be updated. + """ + self._packages = {} + self._num2update = 0 + + # Call system pkg_mnger to get the packages as list of dictionaries. + for pkg in self._pkg_mnger.getPackagesList(): + + # Check if already exist a package in self._packages + pkg_id = pkg.get('package_name') + if pkg_id in self._packages.keys(): + # package already listed to update. do nothing + continue + + # Update the self._packages and self._num2update + self._packages[pkg_id] = pkg + self._num2update = self._num2update + 1 + + def getUpdates(self): + """ + Return the self._packages. + """ + self._scanUpdates() + return self._packages + + def getUpdate(self, name): + """ + Return a dictionary with all info from a given package name. + """ + if name not in self._packages.keys(): + raise NotFoundError('GGBPKGUPD0002E', {'name': name}) + + return self._packages[name] + + def getNumOfUpdates(self): + """ + Return the number of packages to be updated. + """ + self._scanUpdates() + return self._num2update + + def preUpdate(self): + """ + Make adjustments before executing the command in + a child process. + """ + os.setsid() + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + def tailUpdateLogs(self, cb, params): + """ + When the package manager is already running (started outside kimchi or + if wokd is restarted) we can only know what's happening by reading the + logfiles. This method acts like a 'tail -f' on the default package + manager logfile. If the logfile is not found, a simple '*' is + displayed to track progress. This will be until the process finishes. + """ + if not self._pkg_mnger.isRunning(): + return + + fd = None + try: + fd = os.open(self._pkg_mnger.logfile, os.O_RDONLY) + + # cannot open logfile, print something to let users know that the + # system is being upgrading until the package manager finishes its + # job + except (TypeError, OSError): + msgs = [] + while self._pkg_mnger.isRunning(): + msgs.append('*') + cb(''.join(msgs)) + time.sleep(1) + msgs.append('\n') + cb(''.join(msgs), True) + return + + # go to the end of logfile and starts reading, if nothing is read or + # a pattern is not found in the message just wait and retry until + # the package manager finishes + os.lseek(fd, 0, os.SEEK_END) + msgs = [] + progress = [] + while True: + read = os.read(fd, 1024) + if not read: + if not self._pkg_mnger.isRunning(): + break + + if not msgs: + progress.append('*') + cb(''.join(progress)) + + time.sleep(1) + continue + + msgs.append(read) + cb(''.join(msgs)) + + os.close(fd) + return cb(''.join(msgs), True) + + def doUpdate(self, cb, params): + """ + Execute the update + """ + # reset messages + cb('') + + cmd = self._pkg_mnger.update_cmd + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=self.preUpdate) + msgs = [] + while proc.poll() is None: + msgs.append(proc.stdout.readline()) + cb(''.join(msgs)) + time.sleep(0.5) + + # read the final output lines + msgs.extend(proc.stdout.readlines()) + + retcode = proc.poll() + if retcode == 0: + return cb(''.join(msgs), True) + + msgs.extend(proc.stderr.readlines()) + return cb(''.join(msgs), False) + + +class YumUpdate(object): + """ + Class to represent and operate with YUM software update system. + It's loaded only on those systems listed at YUM_DISTROS and loads necessary + modules in runtime. + """ + def __init__(self): + self._pkgs = {} + self.update_cmd = ["yum", "-y", "update"] + self.logfile = self._get_output_log() + + def _get_output_log(self): + """ + Return the logfile path + """ + yumcfg = None + try: + yumcfg = ConfigObj('/etc/yum.conf') + + except ConfigObjError: + return None + + if 'main' in yumcfg and 'logfile' in yumcfg['main']: + return yumcfg['main']['logfile'] + + return None + + def _refreshUpdateList(self): + """ + Update the list of packages to be updated in the system. + """ + try: + gingerBaseLock.acquire() + self._pkgs = get_yum_packages_list_update() + except Exception, e: + raise OperationFailed('GGBPKGUPD0003E', {'err': str(e)}) + finally: + gingerBaseLock.release() + + def getPackagesList(self): + """ + Return a list of package's dictionaries. Each dictionary contains the + information about a package, in the format: + package = {'package_name': <string>, 'version': <string>, + 'arch': <string>, 'repository': <string>} + """ + if self.isRunning(): + raise OperationFailed('GGBPKGUPD0005E') + + self._refreshUpdateList() + pkg_list = [] + for pkg in self._pkgs: + package = {'package_name': pkg.name, 'version': pkg.version, + 'arch': pkg.arch, 'repository': pkg.ui_from_repo} + pkg_list.append(package) + return pkg_list + + def isRunning(self): + """ + Return True whether the YUM package manager is already running or + False otherwise. + """ + try: + with open('/var/run/yum.pid', 'r') as pidfile: + pid = int(pidfile.read().rstrip('\n')) + + # cannot find pidfile, assumes yum is not running + except (IOError, ValueError): + return False + + # the pidfile exists and it lives in process table + if pid_exists(pid): + return True + + return False + + +class AptUpdate(object): + """ + Class to represent and operate with APT software update system. + It's loaded only on those systems listed at APT_DISTROS and loads necessary + modules in runtime. + """ + def __init__(self): + self._pkgs = {} + self.pkg_lock = getattr(__import__('apt_pkg'), 'SystemLock') + self.update_cmd = ['apt-get', 'upgrade', '-y'] + self.logfile = '/var/log/apt/term.log' + + def _refreshUpdateList(self): + """ + Update the list of packages to be updated in the system. + """ + apt_cache = getattr(__import__('apt'), 'Cache')() + try: + with self.pkg_lock(): + apt_cache.update() + apt_cache.upgrade() + self._pkgs = apt_cache.get_changes() + except Exception, e: + gingerBaseLock.release() + raise OperationFailed('GGBPKGUPD0003E', {'err': e.message}) + + def getPackagesList(self): + """ + Return a list of package's dictionaries. Each dictionary contains the + information about a package, in the format + package = {'package_name': <string>, 'version': <string>, + 'arch': <string>, 'repository': <string>} + """ + if self.isRunning(): + raise OperationFailed('GGBPKGUPD0005E') + + gingerBaseLock.acquire() + self._refreshUpdateList() + gingerBaseLock.release() + pkg_list = [] + for pkg in self._pkgs: + package = {'package_name': pkg.shortname, + 'version': pkg.candidate.version, + 'arch': pkg._pkg.architecture, + 'repository': pkg.candidate.origins[0].label} + pkg_list.append(package) + + return pkg_list + + def isRunning(self): + """ + Return True whether the APT package manager is already running or + False otherwise. + """ + try: + with open('/var/lib/dpkg/lock', 'w') as lockfile: + fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) + + # cannot open dpkg lock file to write in exclusive mode means the + # apt is currently running + except IOError: + return True + + return False + + +class ZypperUpdate(object): + """ + Class to represent and operate with Zypper software update system. + It's loaded only on those systems listed at ZYPPER_DISTROS and loads + necessary modules in runtime. + """ + def __init__(self): + self._pkgs = {} + self.update_cmd = ["zypper", "--non-interactive", "update", + "--auto-agree-with-licenses"] + self.logfile = '/var/log/zypp/history' + + def _refreshUpdateList(self): + """ + Update the list of packages to be updated in the system. + """ + self._pkgs = [] + cmd = ["zypper", "list-updates"] + (stdout, stderr, returncode) = run_command(cmd) + + if len(stderr) > 0: + raise OperationFailed('GGBPKGUPD0003E', {'err': stderr}) + + for line in stdout.split('\n'): + if line.find('v |') >= 0: + info = line.split(' | ') + package = {'package_name': info[2], 'version': info[4], + 'arch': info[5], 'repository': info[1]} + self._pkgs.append(package) + + def getPackagesList(self): + """ + Return a list of package's dictionaries. Each dictionary contains the + information about a package, in the format + package = {'package_name': <string>, 'version': <string>, + 'arch': <string>, 'repository': <string>} + """ + if self.isRunning(): + raise OperationFailed('GGBPKGUPD0005E') + + gingerBaseLock.acquire() + self._refreshUpdateList() + gingerBaseLock.release() + return self._pkgs + + def isRunning(self): + """ + Return True whether the Zypper package manager is already running or + False otherwise. + """ + try: + with open('/var/run/zypp.pid', 'r') as pidfile: + pid = int(pidfile.read().rstrip('\n')) + + # cannot find pidfile, assumes yum is not running + except (IOError, ValueError): + return False + + # the pidfile exists and it lives in process table + if pid_exists(pid): + return True + + return False diff --git a/src/wok/plugins/gingerbase/utils.py b/src/wok/plugins/gingerbase/utils.py new file mode 100644 index 0000000..9f41967 --- /dev/null +++ b/src/wok/plugins/gingerbase/utils.py @@ -0,0 +1,82 @@ +# +# Project Kimchi +# +# Copyright IBM, Corp. 2013-2015 +# +# Code derived from Project Kimchi +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# + +import contextlib +import os +import urllib2 +from httplib import HTTPConnection, HTTPException +from urlparse import urlparse + +from wok.exception import InvalidParameter + + +MAX_REDIRECTION_ALLOWED = 5 + + +def check_url_path(path, redirected=0): + if redirected > MAX_REDIRECTION_ALLOWED: + return False + try: + code = '' + parse_result = urlparse(path) + server_name = parse_result.netloc + urlpath = parse_result.path + if not urlpath: + # Just a server, as with a repo. + with contextlib.closing(urllib2.urlopen(path)) as res: + code = res.getcode() + else: + # socket.gaierror could be raised, + # which is a child class of IOError + conn = HTTPConnection(server_name, timeout=15) + # Don't try to get the whole file: + conn.request('HEAD', path) + response = conn.getresponse() + code = response.status + conn.close() + if code == 200: + return True + elif code == 301 or code == 302: + for header in response.getheaders(): + if header[0] == 'location': + return check_url_path(header[1], redirected+1) + else: + return False + except (urllib2.URLError, HTTPException, IOError, ValueError): + return False + return True + + +def validate_repo_url(url): + url_parts = url.split('://') # [0] = prefix, [1] = rest of URL + + if url_parts[0] == '': + raise InvalidParameter("KCHREPOS0002E") + + if url_parts[0] in ['http', 'https', 'ftp']: + if not check_url_path(url): + raise InvalidParameter("WOKUTILS0001E", {'url': url}) + elif url_parts[0] == 'file': + if not os.path.exists(url_parts[1]): + raise InvalidParameter("WOKUTILS0001E", {'url': url}) + else: + raise InvalidParameter("KCHREPOS0002E") diff --git a/src/wok/plugins/kimchi/swupdate.py b/src/wok/plugins/kimchi/swupdate.py deleted file mode 100644 index aba53e6..0000000 --- a/src/wok/plugins/kimchi/swupdate.py +++ /dev/null @@ -1,412 +0,0 @@ -# -# Project Kimchi -# -# Copyright IBM, Corp. 2014-2015 -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -import fcntl -import os -import signal -import subprocess -import time -from configobj import ConfigObj, ConfigObjError -from psutil import pid_exists - -from wok.basemodel import Singleton -from wok.exception import NotFoundError, OperationFailed -from wok.utils import run_command, wok_log - -from wok.plugins.kimchi.config import kimchiLock -from wok.plugins.kimchi.yumparser import get_yum_packages_list_update - - -class SoftwareUpdate(object): - __metaclass__ = Singleton - - """ - Class to represent and operate with OS software update. - """ - def __init__(self): - # This stores all packages to be updated for Kimchi perspective. It's a - # dictionary of dictionaries, in the format {'package_name': package}, - # where: - # package = {'package_name': <string>, 'version': <string>, - # 'arch': <string>, 'repository': <string> - # } - self._packages = {} - - # This stores the number of packages to update - self._num2update = 0 - - # Get the distro of host machine and creates an object related to - # correct package management system - try: - __import__('yum') - wok_log.info("Loading YumUpdate features.") - self._pkg_mnger = YumUpdate() - except ImportError: - try: - __import__('apt') - wok_log.info("Loading AptUpdate features.") - self._pkg_mnger = AptUpdate() - except ImportError: - zypper_help = ["zypper", "--help"] - (stdout, stderr, returncode) = run_command(zypper_help) - if returncode == 0: - wok_log.info("Loading ZypperUpdate features.") - self._pkg_mnger = ZypperUpdate() - else: - raise Exception("There is no compatible package manager " - "for this system.") - - def _scanUpdates(self): - """ - Update self._packages with packages to be updated. - """ - self._packages = {} - self._num2update = 0 - - # Call system pkg_mnger to get the packages as list of dictionaries. - for pkg in self._pkg_mnger.getPackagesList(): - - # Check if already exist a package in self._packages - pkg_id = pkg.get('package_name') - if pkg_id in self._packages.keys(): - # package already listed to update. do nothing - continue - - # Update the self._packages and self._num2update - self._packages[pkg_id] = pkg - self._num2update = self._num2update + 1 - - def getUpdates(self): - """ - Return the self._packages. - """ - self._scanUpdates() - return self._packages - - def getUpdate(self, name): - """ - Return a dictionary with all info from a given package name. - """ - if name not in self._packages.keys(): - raise NotFoundError('KCHPKGUPD0002E', {'name': name}) - - return self._packages[name] - - def getNumOfUpdates(self): - """ - Return the number of packages to be updated. - """ - self._scanUpdates() - return self._num2update - - def preUpdate(self): - """ - Make adjustments before executing the command in - a child process. - """ - os.setsid() - signal.signal(signal.SIGTERM, signal.SIG_IGN) - - def tailUpdateLogs(self, cb, params): - """ - When the package manager is already running (started outside kimchi or - if wokd is restarted) we can only know what's happening by reading the - logfiles. This method acts like a 'tail -f' on the default package - manager logfile. If the logfile is not found, a simple '*' is - displayed to track progress. This will be until the process finishes. - """ - if not self._pkg_mnger.isRunning(): - return - - fd = None - try: - fd = os.open(self._pkg_mnger.logfile, os.O_RDONLY) - - # cannot open logfile, print something to let users know that the - # system is being upgrading until the package manager finishes its - # job - except (TypeError, OSError): - msgs = [] - while self._pkg_mnger.isRunning(): - msgs.append('*') - cb(''.join(msgs)) - time.sleep(1) - msgs.append('\n') - cb(''.join(msgs), True) - return - - # go to the end of logfile and starts reading, if nothing is read or - # a pattern is not found in the message just wait and retry until - # the package manager finishes - os.lseek(fd, 0, os.SEEK_END) - msgs = [] - progress = [] - while True: - read = os.read(fd, 1024) - if not read: - if not self._pkg_mnger.isRunning(): - break - - if not msgs: - progress.append('*') - cb(''.join(progress)) - - time.sleep(1) - continue - - msgs.append(read) - cb(''.join(msgs)) - - os.close(fd) - return cb(''.join(msgs), True) - - def doUpdate(self, cb, params): - """ - Execute the update - """ - # reset messages - cb('') - - cmd = self._pkg_mnger.update_cmd - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=self.preUpdate) - msgs = [] - while proc.poll() is None: - msgs.append(proc.stdout.readline()) - cb(''.join(msgs)) - time.sleep(0.5) - - # read the final output lines - msgs.extend(proc.stdout.readlines()) - - retcode = proc.poll() - if retcode == 0: - return cb(''.join(msgs), True) - - msgs.extend(proc.stderr.readlines()) - return cb(''.join(msgs), False) - - -class YumUpdate(object): - """ - Class to represent and operate with YUM software update system. - It's loaded only on those systems listed at YUM_DISTROS and loads necessary - modules in runtime. - """ - def __init__(self): - self._pkgs = {} - self.update_cmd = ["yum", "-y", "update"] - self.logfile = self._get_output_log() - - def _get_output_log(self): - """ - Return the logfile path - """ - yumcfg = None - try: - yumcfg = ConfigObj('/etc/yum.conf') - - except ConfigObjError: - return None - - if 'main' in yumcfg and 'logfile' in yumcfg['main']: - return yumcfg['main']['logfile'] - - return None - - def _refreshUpdateList(self): - """ - Update the list of packages to be updated in the system. - """ - try: - kimchiLock.acquire() - self._pkgs = get_yum_packages_list_update() - except Exception, e: - raise OperationFailed('KCHPKGUPD0003E', {'err': str(e)}) - finally: - kimchiLock.release() - - def getPackagesList(self): - """ - Return a list of package's dictionaries. Each dictionary contains the - information about a package, in the format: - package = {'package_name': <string>, 'version': <string>, - 'arch': <string>, 'repository': <string>} - """ - if self.isRunning(): - raise OperationFailed('KCHPKGUPD0005E') - - self._refreshUpdateList() - pkg_list = [] - for pkg in self._pkgs: - package = {'package_name': pkg.name, 'version': pkg.version, - 'arch': pkg.arch, 'repository': pkg.ui_from_repo} - pkg_list.append(package) - return pkg_list - - def isRunning(self): - """ - Return True whether the YUM package manager is already running or - False otherwise. - """ - try: - with open('/var/run/yum.pid', 'r') as pidfile: - pid = int(pidfile.read().rstrip('\n')) - - # cannot find pidfile, assumes yum is not running - except (IOError, ValueError): - return False - - # the pidfile exists and it lives in process table - if pid_exists(pid): - return True - - return False - - -class AptUpdate(object): - """ - Class to represent and operate with APT software update system. - It's loaded only on those systems listed at APT_DISTROS and loads necessary - modules in runtime. - """ - def __init__(self): - self._pkgs = {} - self.pkg_lock = getattr(__import__('apt_pkg'), 'SystemLock') - self.update_cmd = ['apt-get', 'upgrade', '-y'] - self.logfile = '/var/log/apt/term.log' - - def _refreshUpdateList(self): - """ - Update the list of packages to be updated in the system. - """ - apt_cache = getattr(__import__('apt'), 'Cache')() - try: - with self.pkg_lock(): - apt_cache.update() - apt_cache.upgrade() - self._pkgs = apt_cache.get_changes() - except Exception, e: - kimchiLock.release() - raise OperationFailed('KCHPKGUPD0003E', {'err': e.message}) - - def getPackagesList(self): - """ - Return a list of package's dictionaries. Each dictionary contains the - information about a package, in the format - package = {'package_name': <string>, 'version': <string>, - 'arch': <string>, 'repository': <string>} - """ - if self.isRunning(): - raise OperationFailed('KCHPKGUPD0005E') - - kimchiLock.acquire() - self._refreshUpdateList() - kimchiLock.release() - pkg_list = [] - for pkg in self._pkgs: - package = {'package_name': pkg.shortname, - 'version': pkg.candidate.version, - 'arch': pkg._pkg.architecture, - 'repository': pkg.candidate.origins[0].label} - pkg_list.append(package) - - return pkg_list - - def isRunning(self): - """ - Return True whether the APT package manager is already running or - False otherwise. - """ - try: - with open('/var/lib/dpkg/lock', 'w') as lockfile: - fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - # cannot open dpkg lock file to write in exclusive mode means the - # apt is currently running - except IOError: - return True - - return False - - -class ZypperUpdate(object): - """ - Class to represent and operate with Zypper software update system. - It's loaded only on those systems listed at ZYPPER_DISTROS and loads - necessary modules in runtime. - """ - def __init__(self): - self._pkgs = {} - self.update_cmd = ["zypper", "--non-interactive", "update", - "--auto-agree-with-licenses"] - self.logfile = '/var/log/zypp/history' - - def _refreshUpdateList(self): - """ - Update the list of packages to be updated in the system. - """ - self._pkgs = [] - cmd = ["zypper", "list-updates"] - (stdout, stderr, returncode) = run_command(cmd) - - if len(stderr) > 0: - raise OperationFailed('KCHPKGUPD0003E', {'err': stderr}) - - for line in stdout.split('\n'): - if line.find('v |') >= 0: - info = line.split(' | ') - package = {'package_name': info[2], 'version': info[4], - 'arch': info[5], 'repository': info[1]} - self._pkgs.append(package) - - def getPackagesList(self): - """ - Return a list of package's dictionaries. Each dictionary contains the - information about a package, in the format - package = {'package_name': <string>, 'version': <string>, - 'arch': <string>, 'repository': <string>} - """ - if self.isRunning(): - raise OperationFailed('KCHPKGUPD0005E') - - kimchiLock.acquire() - self._refreshUpdateList() - kimchiLock.release() - return self._pkgs - - def isRunning(self): - """ - Return True whether the Zypper package manager is already running or - False otherwise. - """ - try: - with open('/var/run/zypp.pid', 'r') as pidfile: - pid = int(pidfile.read().rstrip('\n')) - - # cannot find pidfile, assumes yum is not running - except (IOError, ValueError): - return False - - # the pidfile exists and it lives in process table - if pid_exists(pid): - return True - - return False -- 2.1.0