[Kimchi-devel] [PATCH 03/17] V7 Ginger Base : base folder files part 2
chandra at linux.vnet.ibm.com
chandra at linux.vnet.ibm.com
Wed Oct 21 11:10:48 UTC 2015
From: chandrureddy <chandra at linux.vnet.ibm.com>
---
src/wok/plugins/gingerbase/disks.py | 198 +++++++++++
src/wok/plugins/gingerbase/repositories.py | 538 +++++++++++++++++++++++++++++
src/wok/plugins/gingerbase/yumparser.py | 355 +++++++++++++++++++
src/wok/plugins/kimchi/disks.py | 196 -----------
src/wok/plugins/kimchi/repositories.py | 534 ----------------------------
src/wok/plugins/kimchi/yumparser.py | 353 -------------------
6 files changed, 1091 insertions(+), 1083 deletions(-)
create mode 100644 src/wok/plugins/gingerbase/disks.py
create mode 100644 src/wok/plugins/gingerbase/repositories.py
create mode 100644 src/wok/plugins/gingerbase/yumparser.py
delete mode 100644 src/wok/plugins/kimchi/disks.py
delete mode 100644 src/wok/plugins/kimchi/repositories.py
delete mode 100644 src/wok/plugins/kimchi/yumparser.py
diff --git a/src/wok/plugins/gingerbase/disks.py b/src/wok/plugins/gingerbase/disks.py
new file mode 100644
index 0000000..234e385
--- /dev/null
+++ b/src/wok/plugins/gingerbase/disks.py
@@ -0,0 +1,198 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2013-2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import os.path
+import re
+import subprocess
+from parted import Device as PDevice
+from parted import Disk as PDisk
+
+from wok.exception import OperationFailed
+from wok.utils import wok_log
+
+
+def _get_dev_node_path(maj_min):
+ """ Returns device node path given the device number 'major:min' """
+
+ dm_name = "/sys/dev/block/%s/dm/name" % maj_min
+ if os.path.exists(dm_name):
+ with open(dm_name) as dm_f:
+ content = dm_f.read().rstrip('\n')
+ return "/dev/mapper/" + content
+
+ uevent = "/sys/dev/block/%s/uevent" % maj_min
+ with open(uevent) as ueventf:
+ content = ueventf.read()
+
+ data = dict(re.findall(r'(\S+)=(".*?"|\S+)', content.replace("\n", " ")))
+
+ return "/dev/%s" % data["DEVNAME"]
+
+
+def _get_lsblk_devs(keys, devs=[]):
+ lsblk = subprocess.Popen(
+ ["lsblk", "-Pbo"] + [','.join(keys)] + devs,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = lsblk.communicate()
+ if lsblk.returncode != 0:
+ raise OperationFailed("GGBDISKS0001E", {'err': err})
+
+ return _parse_lsblk_output(out, keys)
+
+
+def _get_dev_major_min(name):
+ maj_min = None
+
+ keys = ["NAME", "MAJ:MIN"]
+ dev_list = _get_lsblk_devs(keys)
+
+ for dev in dev_list:
+ if dev['name'].split()[0] == name:
+ maj_min = dev['maj:min']
+ break
+ else:
+ raise OperationFailed("GGBDISKS0002E", {'device': name})
+
+ return maj_min
+
+
+def _is_dev_leaf(devNodePath):
+ try:
+ # By default, lsblk prints a device information followed by children
+ # device information
+ childrenCount = len(
+ _get_lsblk_devs(["NAME"], [devNodePath])) - 1
+ except OperationFailed as e:
+ # lsblk is known to fail on multipath devices
+ # Assume these devices contain children
+ wok_log.error(
+ "Error getting device info for %s: %s", devNodePath, e)
+ return False
+
+ return childrenCount == 0
+
+
+def _is_dev_extended_partition(devType, devNodePath):
+ if devType != 'part':
+ return False
+ diskPath = devNodePath.rstrip('0123456789')
+ device = PDevice(diskPath)
+ try:
+ extended_part = PDisk(device).getExtendedPartition()
+ except NotImplementedError as e:
+ wok_log.warning(
+ "Error getting extended partition info for dev %s type %s: %s",
+ devNodePath, devType, e.message)
+ # Treate disk with unsupported partiton table as if it does not
+ # contain extended partitions.
+ return False
+ if extended_part and extended_part.path == devNodePath:
+ return True
+ return False
+
+
+def _parse_lsblk_output(output, keys):
+ # output is on format key="value",
+ # where key can be NAME, TYPE, FSTYPE, SIZE, MOUNTPOINT, etc
+ lines = output.rstrip("\n").split("\n")
+ r = []
+ for line in lines:
+ d = {}
+ for key in keys:
+ expression = r"%s=\".*?\"" % key
+ match = re.search(expression, line)
+ field = match.group()
+ k, v = field.split('=', 1)
+ d[k.lower()] = v[1:-1]
+ r.append(d)
+ return r
+
+
+def _get_vgname(devNodePath):
+ """ Return volume group name of a physical volume. If the device node path
+ is not a physical volume, return empty string. """
+ pvs = subprocess.Popen(
+ ["pvs", "--unbuffered", "--nameprefixes", "--noheadings",
+ "-o", "vg_name", devNodePath],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = pvs.communicate()
+ if pvs.returncode != 0:
+ return ""
+
+ return re.findall(r"LVM2_VG_NAME='([^\']*)'", out)[0]
+
+
+def _is_available(name, devtype, fstype, mountpoint, majmin):
+ devNodePath = _get_dev_node_path(majmin)
+ # Only list unmounted and unformated and leaf and (partition or disk)
+ # leaf means a partition, a disk has no partition, or a disk not held
+ # by any multipath device. Physical volume belongs to no volume group
+ # is also listed. Extended partitions should not be listed.
+ if (devtype in ['part', 'disk', 'mpath'] and
+ fstype in ['', 'LVM2_member'] and
+ mountpoint == "" and
+ _get_vgname(devNodePath) == "" and
+ _is_dev_leaf(devNodePath) and
+ not _is_dev_extended_partition(devtype, devNodePath)):
+ return True
+ return False
+
+
+def get_partitions_names(check=False):
+ names = set()
+ keys = ["NAME", "TYPE", "FSTYPE", "MOUNTPOINT", "MAJ:MIN"]
+ # output is on format key="value",
+ # where key can be NAME, TYPE, FSTYPE, MOUNTPOINT
+ for dev in _get_lsblk_devs(keys):
+ # split()[0] to avoid the second part of the name, after the
+ # whiteline
+ name = dev['name'].split()[0]
+ if check and not _is_available(name, dev['type'], dev['fstype'],
+ dev['mountpoint'], dev['maj:min']):
+ continue
+ names.add(name)
+
+ return list(names)
+
+
+def get_partition_details(name):
+ majmin = _get_dev_major_min(name)
+ dev_path = _get_dev_node_path(majmin)
+
+ keys = ["TYPE", "FSTYPE", "SIZE", "MOUNTPOINT"]
+ try:
+ dev = _get_lsblk_devs(keys, [dev_path])[0]
+ except OperationFailed as e:
+ wok_log.error(
+ "Error getting partition info for %s: %s", name, e)
+ return {}
+
+ dev['available'] = _is_available(name, dev['type'], dev['fstype'],
+ dev['mountpoint'], majmin)
+ if dev['mountpoint']:
+ # Sometimes the mountpoint comes with [SWAP] or other
+ # info which is not an actual mount point. Filtering it
+ regexp = re.compile(r"\[.*\]")
+ if regexp.search(dev['mountpoint']) is not None:
+ dev['mountpoint'] = ''
+ dev['path'] = dev_path
+ dev['name'] = name
+ return dev
diff --git a/src/wok/plugins/gingerbase/repositories.py b/src/wok/plugins/gingerbase/repositories.py
new file mode 100644
index 0000000..146d019
--- /dev/null
+++ b/src/wok/plugins/gingerbase/repositories.py
@@ -0,0 +1,538 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2014-2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import copy
+import os
+import time
+import urlparse
+from ConfigParser import ConfigParser
+
+from wok.basemodel import Singleton
+from wok.exception import InvalidOperation, InvalidParameter
+from wok.exception import OperationFailed, NotFoundError, MissingParameter
+
+from wok.plugins.gingerbase.config import gingerBaseLock
+from wok.plugins.gingerbase.utils import validate_repo_url
+from wok.plugins.gingerbase.yumparser import get_yum_repositories
+from wok.plugins.gingerbase.yumparser import write_repo_to_file
+from wok.plugins.gingerbase.yumparser import get_display_name
+from wok.plugins.gingerbase.yumparser import get_expanded_url
+
+
+class Repositories(object):
+ __metaclass__ = Singleton
+
+ """
+ Class to represent and operate with repositories information.
+ """
+ def __init__(self):
+ try:
+ __import__('yum')
+ self._pkg_mnger = YumRepo()
+ except ImportError:
+ try:
+ __import__('apt_pkg')
+ self._pkg_mnger = AptRepo()
+ except ImportError:
+ raise InvalidOperation('GGBREPOS0014E')
+
+ def addRepository(self, params):
+ """
+ Add and enable a new repository
+ """
+ config = params.get('config', {})
+ extra_keys = list(
+ set(config.keys()).difference(set(self._pkg_mnger.CONFIG_ENTRY)))
+ if len(extra_keys) > 0:
+ raise InvalidParameter("GGBREPOS0028E",
+ {'items': ",".join(extra_keys)})
+
+ return self._pkg_mnger.addRepo(params)
+
+ def getRepositories(self):
+ """
+ Return a dictionary with all Ginger Base repositories. Each element
+ uses the format {<repo_id>: {repo}}, where repo is a dictionary in the
+ repositories.Repositories() format.
+ """
+ return self._pkg_mnger.getRepositoriesList()
+
+ def getRepository(self, repo_id):
+ """
+ Return a dictionary with all info from a given repository ID.
+ """
+ info = self._pkg_mnger.getRepo(repo_id)
+ info['repo_id'] = repo_id
+ return info
+
+ def enableRepository(self, repo_id):
+ """
+ Enable a repository.
+ """
+ return self._pkg_mnger.toggleRepo(repo_id, True)
+
+ def disableRepository(self, repo_id):
+ """
+ Disable a given repository.
+ """
+ return self._pkg_mnger.toggleRepo(repo_id, False)
+
+ def updateRepository(self, repo_id, params):
+ """
+ Update the information of a given repository.
+ The input is the repo_id of the repository to be updated and a dict
+ with the information to be updated.
+ """
+ return self._pkg_mnger.updateRepo(repo_id, params)
+
+ def removeRepository(self, repo_id):
+ """
+ Remove a given repository
+ """
+ return self._pkg_mnger.removeRepo(repo_id)
+
+
+class YumRepo(object):
+ """
+ Class to represent and operate with YUM repositories.
+ It's loaded only on those systems listed at YUM_DISTROS and loads necessary
+ modules in runtime.
+ """
+ TYPE = 'yum'
+ DEFAULT_CONF_DIR = "/etc/yum.repos.d"
+ CONFIG_ENTRY = ('repo_name', 'mirrorlist', 'metalink')
+
+ def __init__(self):
+ self._confdir = self.DEFAULT_CONF_DIR
+
+ def _get_repos(self, errcode):
+ try:
+ gingerBaseLock.acquire()
+ repos = get_yum_repositories()
+ except Exception, e:
+ gingerBaseLock.release()
+ raise OperationFailed(errcode, {'err': str(e)})
+ finally:
+ gingerBaseLock.release()
+
+ return repos
+
+ def getRepositoriesList(self):
+ """
+ Return a list of repositories IDs
+ """
+ repos = self._get_repos('GGBREPOS0024E')
+ return repos.keys()
+
+ def getRepo(self, repo_id):
+ """
+ Return a dictionary in the repositories.Repositories() of the given
+ repository ID format with the information of a YumRepository object.
+ """
+ repos = self._get_repos('GGBREPOS0025E')
+
+ if repo_id not in repos.keys():
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ entry = repos.get(repo_id)
+
+ display_name = get_display_name(entry.name)
+
+ info = {}
+ info['enabled'] = entry.enabled
+ info['baseurl'] = entry.baseurl or ''
+ info['config'] = {}
+ info['config']['display_repo_name'] = display_name
+ info['config']['repo_name'] = entry.name or ''
+ info['config']['gpgcheck'] = entry.gpgcheck
+ info['config']['gpgkey'] = entry.gpgkey or ''
+ info['config']['mirrorlist'] = entry.mirrorlist or ''
+ info['config']['metalink'] = entry.metalink or ''
+ return info
+
+ def addRepo(self, params):
+ """
+ Add a given repository to YumBase
+ """
+ # At least one base url, or one mirror, must be given.
+ baseurl = params.get('baseurl', '')
+
+ config = params.get('config', {})
+ mirrorlist = config.get('mirrorlist', '')
+ metalink = config.get('metalink', '')
+ if not baseurl and not mirrorlist and not metalink:
+ raise MissingParameter("GGBREPOS0013E")
+
+ if baseurl:
+ validate_repo_url(get_expanded_url(baseurl))
+
+ if mirrorlist:
+ validate_repo_url(get_expanded_url(mirrorlist))
+
+ if metalink:
+ validate_repo_url(get_expanded_url(metalink))
+
+ if mirrorlist and metalink:
+ raise InvalidOperation('GGBREPOS0030E')
+
+ repo_id = params.get('repo_id', None)
+ if repo_id is None:
+ repo_id = "gingerbase_repo_%s" % str(int(time.time() * 1000))
+
+ repos = self._get_repos('GGBREPOS0026E')
+ if repo_id in repos.keys():
+ raise InvalidOperation("GGBREPOS0022E", {'repo_id': repo_id})
+
+ repo_name = config.get('repo_name', repo_id)
+ repo = {'baseurl': baseurl, 'mirrorlist': mirrorlist,
+ 'name': repo_name, 'gpgcheck': 1,
+ 'gpgkey': [], 'enabled': 1, 'metalink': metalink}
+
+ # write a repo file in the system with repo{} information.
+ parser = ConfigParser()
+ parser.add_section(repo_id)
+
+ for key, value in repo.iteritems():
+ if value:
+ parser.set(repo_id, key, value)
+
+ repofile = os.path.join(self._confdir, repo_id + '.repo')
+ try:
+ with open(repofile, 'w') as fd:
+ parser.write(fd)
+ except:
+ raise OperationFailed("GGBREPOS0018E",
+ {'repo_file': repofile})
+
+ return repo_id
+
+ def toggleRepo(self, repo_id, enable):
+ repos = self._get_repos('GGBREPOS0011E')
+ if repo_id not in repos.keys():
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ entry = repos.get(repo_id)
+ if enable and entry.enabled:
+ raise InvalidOperation("GGBREPOS0015E", {'repo_id': repo_id})
+
+ if not enable and not entry.enabled:
+ raise InvalidOperation("GGBREPOS0016E", {'repo_id': repo_id})
+
+ gingerBaseLock.acquire()
+ try:
+ if enable:
+ entry.enable()
+ else:
+ entry.disable()
+
+ write_repo_to_file(entry)
+ except:
+ if enable:
+ raise OperationFailed("GGBREPOS0020E", {'repo_id': repo_id})
+
+ raise OperationFailed("GGBREPOS0021E", {'repo_id': repo_id})
+ finally:
+ gingerBaseLock.release()
+
+ return repo_id
+
+ def updateRepo(self, repo_id, params):
+ """
+ Update a given repository in repositories.Repositories() format
+ """
+ repos = self._get_repos('GGBREPOS0011E')
+ if repo_id not in repos.keys():
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ entry = repos.get(repo_id)
+
+ baseurl = params.get('baseurl', None)
+ config = params.get('config', {})
+ mirrorlist = config.get('mirrorlist', None)
+ metalink = config.get('metalink', None)
+
+ if baseurl is not None and len(baseurl.strip()) == 0:
+ baseurl = None
+
+ if mirrorlist is not None and len(mirrorlist.strip()) == 0:
+ mirrorlist = None
+
+ if metalink is not None and len(metalink.strip()) == 0:
+ metalink = None
+
+ if baseurl is None and mirrorlist is None and metalink is None:
+ raise MissingParameter("GGBREPOS0013E")
+
+ if baseurl is not None:
+ validate_repo_url(get_expanded_url(baseurl))
+ entry.baseurl = baseurl
+
+ if mirrorlist is not None:
+ validate_repo_url(get_expanded_url(mirrorlist))
+ entry.mirrorlist = mirrorlist
+
+ if metalink is not None:
+ validate_repo_url(get_expanded_url(metalink))
+ entry.metalink = metalink
+
+ if mirrorlist and metalink:
+ raise InvalidOperation('GGBREPOS0030E')
+
+ entry.id = params.get('repo_id', repo_id)
+ entry.name = config.get('repo_name', entry.name)
+ entry.gpgcheck = config.get('gpgcheck', entry.gpgcheck)
+ entry.gpgkey = config.get('gpgkey', entry.gpgkey)
+ gingerBaseLock.acquire()
+ write_repo_to_file(entry)
+ gingerBaseLock.release()
+ return repo_id
+
+ def removeRepo(self, repo_id):
+ """
+ Remove a given repository
+ """
+ repos = self._get_repos('GGBREPOS0027E')
+ if repo_id not in repos.keys():
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ entry = repos.get(repo_id)
+ parser = ConfigParser()
+ with open(entry.repofile) as fd:
+ parser.readfp(fd)
+
+ if len(parser.sections()) == 1:
+ os.remove(entry.repofile)
+ return
+
+ parser.remove_section(repo_id)
+ with open(entry.repofile, "w") as fd:
+ parser.write(fd)
+
+
+class AptRepo(object):
+ """
+ Class to represent and operate with YUM repositories.
+ It's loaded only on those systems listed at YUM_DISTROS and loads necessary
+ modules in runtime.
+ """
+ TYPE = 'deb'
+ GINGERBASE_LIST = "gingerbase-source.list"
+ CONFIG_ENTRY = ('dist', 'comps')
+
+ def __init__(self):
+ getattr(__import__('apt_pkg'), 'init_config')()
+ getattr(__import__('apt_pkg'), 'init_system')()
+ config = getattr(__import__('apt_pkg'), 'config')
+ self.pkg_lock = getattr(__import__('apt_pkg'), 'SystemLock')
+ module = __import__('aptsources.sourceslist', globals(), locals(),
+ ['SourcesList'], -1)
+
+ self._sourceparts_path = '/%s%s' % (
+ config.get('Dir::Etc'), config.get('Dir::Etc::sourceparts'))
+ self._sourceslist = getattr(module, 'SourcesList')
+ self.filename = os.path.join(self._sourceparts_path,
+ self.GINGERBASE_LIST)
+ if not os.path.exists(self.filename):
+ with open(self.filename, 'w') as fd:
+ fd.write("# This file is managed by Ginger Base and it "
+ "must not be modified manually\n")
+
+ def _get_repos(self):
+ try:
+ with self.pkg_lock():
+ repos = self._sourceslist()
+ repos.refresh()
+ except Exception, e:
+ gingerBaseLock.release()
+ raise OperationFailed('GGBREPOS0025E', {'err': e.message})
+
+ return repos
+
+ def _get_repo_id(self, repo):
+ data = urlparse.urlparse(repo.uri)
+ name = data.hostname or data.path
+ return '%s-%s-%s' % (name, repo.dist, "-".join(repo.comps))
+
+ def _get_source_entry(self, repo_id):
+ gingerBaseLock.acquire()
+ repos = self._get_repos()
+ gingerBaseLock.release()
+
+ for r in repos:
+ # Ignore deb-src repositories
+ if r.type != 'deb':
+ continue
+
+ if self._get_repo_id(r) != repo_id:
+ continue
+
+ return r
+
+ return None
+
+ def getRepositoriesList(self):
+ """
+ Return a list of repositories IDs
+
+ APT repositories there aren't the concept about repository ID, so for
+ internal control, the repository ID will be built as described in
+ _get_repo_id()
+ """
+ gingerBaseLock.acquire()
+ repos = self._get_repos()
+ gingerBaseLock.release()
+
+ res = []
+ for r in repos:
+ # Ignore deb-src repositories
+ if r.type != 'deb':
+ continue
+
+ res.append(self._get_repo_id(r))
+
+ return res
+
+ def getRepo(self, repo_id):
+ """
+ Return a dictionary in the repositories.Repositories() format of the
+ given repository ID with the information of a SourceEntry object.
+ """
+ r = self._get_source_entry(repo_id)
+ if r is None:
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ info = {'enabled': not r.disabled,
+ 'baseurl': r.uri,
+ 'config': {'dist': r.dist,
+ 'comps': r.comps}}
+ return info
+
+ def addRepo(self, params):
+ """
+ Add a new APT repository based on <params>
+ """
+ # To create a APT repository the dist is a required parameter
+ # (in addition to baseurl, verified on controller through API.json)
+ config = params.get('config', None)
+ if config is None:
+ raise MissingParameter("GGBREPOS0019E")
+
+ if 'dist' not in config.keys():
+ raise MissingParameter("GGBREPOS0019E")
+
+ uri = params['baseurl']
+ dist = config['dist']
+ comps = config.get('comps', [])
+
+ validate_repo_url(get_expanded_url(uri))
+
+ gingerBaseLock.acquire()
+ try:
+ repos = self._get_repos()
+ source_entry = repos.add('deb', uri, dist, comps,
+ file=self.filename)
+ with self.pkg_lock():
+ repos.save()
+ except Exception as e:
+ gingerBaseLock.release()
+ raise OperationFailed("GGBREPOS0026E", {'err': e.message})
+ gingerBaseLock.release()
+ return self._get_repo_id(source_entry)
+
+ def toggleRepo(self, repo_id, enable):
+ """
+ Enable a given repository
+ """
+ r = self._get_source_entry(repo_id)
+ if r is None:
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ if enable and not r.disabled:
+ raise InvalidOperation("GGBREPOS0015E", {'repo_id': repo_id})
+
+ if not enable and r.disabled:
+ raise InvalidOperation("GGBREPOS0016E", {'repo_id': repo_id})
+
+ if enable:
+ line = 'deb'
+ else:
+ line = '#deb'
+
+ gingerBaseLock.acquire()
+ try:
+ repos = self._get_repos()
+ with self.pkg_lock():
+ repos.remove(r)
+ repos.add(line, r.uri, r.dist, r.comps, file=self.filename)
+ repos.save()
+ except:
+ gingerBaseLock.release()
+ if enable:
+ raise OperationFailed("GGBREPOS0020E", {'repo_id': repo_id})
+
+ raise OperationFailed("GGBREPOS0021E", {'repo_id': repo_id})
+ finally:
+ gingerBaseLock.release()
+
+ return repo_id
+
+ def updateRepo(self, repo_id, params):
+ """
+ Update a given repository in repositories.Repositories() format
+ """
+ old_info = self.getRepo(repo_id)
+ updated_info = copy.deepcopy(old_info)
+ updated_info['baseurl'] = params.get(
+ 'baseurl', updated_info['baseurl'])
+
+ if 'config' in params.keys():
+ config = params['config']
+ updated_info['config']['dist'] = config.get(
+ 'dist', old_info['config']['dist'])
+ updated_info['config']['comps'] = config.get(
+ 'comps', old_info['config']['comps'])
+
+ self.removeRepo(repo_id)
+ try:
+ return self.addRepo(updated_info)
+ except:
+ self.addRepo(old_info)
+ raise
+
+ def removeRepo(self, repo_id):
+ """
+ Remove a given repository
+ """
+ r = self._get_source_entry(repo_id)
+ if r is None:
+ raise NotFoundError("GGBREPOS0012E", {'repo_id': repo_id})
+
+ gingerBaseLock.acquire()
+ try:
+ repos = self._get_repos()
+ with self.pkg_lock():
+ repos.remove(r)
+ repos.save()
+ except:
+ gingerBaseLock.release()
+ raise OperationFailed("GGBREPOS0017E", {'repo_id': repo_id})
+ finally:
+ gingerBaseLock.release()
diff --git a/src/wok/plugins/gingerbase/yumparser.py b/src/wok/plugins/gingerbase/yumparser.py
new file mode 100644
index 0000000..bd670d2
--- /dev/null
+++ b/src/wok/plugins/gingerbase/yumparser.py
@@ -0,0 +1,355 @@
+#
+# Project Ginger Base
+#
+# Copyright IBM, Corp. 2015
+#
+# Code derived from Project Kimchi
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+import subprocess
+import glob
+
+from os import listdir
+from os.path import isfile, splitext, basename
+
+try:
+ import rpm
+except ImportError:
+ pass
+
+
+class YumRepoObject(object):
+
+ def __init__(self, repo_id, repofile):
+ self.repo_id = repo_id
+ self.name = None
+ self.baseurl = None
+ self.enabled = True
+ self.gpgcheck = True
+ self.gpgkey = None
+ self.metalink = None
+ self.mirrorlist = None
+ self.repofile = repofile
+ self.string_attrs = ['baseurl', 'gpgkey', 'name',
+ 'metalink', 'mirrorlist']
+ self.boolean_attrs = ['enabled', 'gpgcheck']
+
+ def set_attribute(self, key, strvalue):
+ if key in self.string_attrs:
+ setattr(self, key, strvalue)
+ elif key in self.boolean_attrs:
+ setattr(self, key, (strvalue == '1'))
+
+ def get_attribute_str(self, key):
+ if key not in self.get_attributes():
+ return None
+
+ if key in self.boolean_attrs:
+ str_value = '1' if getattr(self, key) is True else '0'
+ else:
+ str_value = getattr(self, key)
+
+ if str_value is None:
+ return None
+
+ return key + '=' + str_value
+
+ def get_attributes(self):
+ return self.string_attrs + self.boolean_attrs
+
+ def enable(self):
+ self.enabled = True
+
+ def disable(self):
+ self.enabled = False
+
+ def __str__(self):
+ str_obj = '[' + self.repo_id + ']' + '\n'
+ for key in self.get_attributes():
+ if self.get_attribute_str(key) is not None:
+ str_obj += self.get_attribute_str(key) + '\n'
+ return str_obj
+
+
+def get_repo_files():
+ def _is_repository_file(f):
+ _, f_extension = splitext(f)
+ return isfile(f) and (f_extension == '.repo')
+
+ YUM_REPO_DIR = '/etc/yum.repos.d'
+ return [YUM_REPO_DIR+'/'+f for f in listdir(YUM_REPO_DIR)
+ if _is_repository_file(YUM_REPO_DIR+'/'+f)]
+
+
+def _ignore_line_repo_file(line):
+ return line.startswith("#") or '=' not in line
+
+
+def _get_repos_from_file(repo_file):
+ repos_from_file = {}
+ current_repo = None
+ current_repo_id = None
+ with open(repo_file) as f:
+ for line in f.readlines():
+ line = line.strip()
+ if line.startswith("["):
+ if current_repo is not None:
+ repos_from_file[current_repo_id] = current_repo
+ current_repo_id = line.strip('[]')
+ current_repo = YumRepoObject(current_repo_id, repo_file)
+ continue
+ if _ignore_line_repo_file(line):
+ continue
+ key, value = line.split('=', 1)
+ key = key.strip()
+ value = value.strip()
+ current_repo.set_attribute(key, value)
+
+ # add the last repo from file.
+ if current_repo is not None:
+ repos_from_file[current_repo_id] = current_repo
+
+ return repos_from_file
+
+
+def get_yum_repositories():
+ repo_files = get_repo_files()
+ repos = {}
+ for yum_repo in repo_files:
+ repos.update(_get_repos_from_file(yum_repo))
+
+ return repos
+
+
+def _retrieve_repo_line_index(data, repo):
+ repo_entry = '[' + repo.repo_id + ']\n'
+ try:
+ repo_index = data.index(repo_entry)
+ except:
+ return None
+ return repo_index
+
+
+def _update_repo_file_data(data, repo, repo_index):
+ remaining_repo_attrs = repo.get_attributes()
+
+ for i in range(repo_index + 1, len(data)):
+ line = data[i].strip()
+ if line.startswith('['):
+ break
+ if _ignore_line_repo_file(line):
+ continue
+ key, _ = line.split('=', 1)
+ key = key.strip()
+ attr_str = repo.get_attribute_str(key)
+ if attr_str is None:
+ continue
+ remaining_repo_attrs.remove(key)
+ data[i] = attr_str + '\n'
+
+ for attr in remaining_repo_attrs:
+ attr_str = repo.get_attribute_str(attr)
+ if attr_str is None:
+ continue
+ data.insert(repo_index+1, attr_str + '\n')
+
+ return data
+
+
+def write_repo_to_file(repo):
+ with open(repo.repofile) as f:
+ data = f.readlines()
+
+ repo_index = _retrieve_repo_line_index(data, repo)
+ if repo_index is None:
+ return
+
+ data = _update_repo_file_data(data, repo, repo_index)
+
+ with open(repo.repofile, 'w') as f:
+ f.writelines(data)
+
+
+def _get_last_line_repo(data, repo_index):
+ stop_delete_index = None
+ for i in range(repo_index+1, len(data)):
+ line = data[i].strip()
+ if line.startswith('['):
+ stop_delete_index = i - 1
+ break
+ if stop_delete_index is None:
+ stop_delete_index = len(data) - 1
+
+ return stop_delete_index
+
+
+def _remove_repo_file_data(data, repo_index):
+ last_line_repo = _get_last_line_repo(data, repo_index)
+ for i in range(last_line_repo, repo_index - 1, -1):
+ data.pop(i)
+ return data
+
+
+def delete_repo_from_file(repo):
+ with open(repo.repofile) as f:
+ data = f.readlines()
+
+ repo_index = _retrieve_repo_line_index(data, repo)
+ if repo_index is None:
+ return
+
+ data = _remove_repo_file_data(data, repo_index)
+
+ with open(repo.repofile, 'w') as f:
+ f.writelines(data)
+
+
+def _get_releasever():
+ release_file = glob.glob('/etc/*-release')[0]
+ transaction = rpm.TransactionSet()
+ match_iter = transaction.dbMatch('basenames', release_file)
+
+ ret = '%releasever'
+ try:
+ ret = match_iter.next()['version']
+
+ except StopIteration:
+ pass
+
+ return ret
+
+
+def _get_basearch():
+ cmd = ['uname', '-i']
+ uname = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ return uname.communicate()[0].strip('"\n')
+
+
+def _get_all_yum_vars():
+ variables = {}
+
+ def _get_var_content(varfile):
+ with open(varfile) as f:
+ variables[basename(varfile)] = f.read().strip('\n')
+
+ map(lambda vfile:
+ _get_var_content(vfile),
+ glob.glob('/etc/yum/vars/*'))
+
+ return variables
+
+
+def _expand_variables(stringvar, split_char=' '):
+ yum_variables = _get_all_yum_vars()
+ yum_variables['releasever'] = _get_releasever()
+ yum_variables['basearch'] = _get_basearch()
+
+ name_vars = [var for var in stringvar.split(split_char)
+ if var.startswith('$') and var.strip('$') in yum_variables]
+
+ return reduce(lambda nm, var:
+ nm.replace(var, yum_variables[var.strip('$')]),
+ name_vars,
+ stringvar)
+
+
+def get_display_name(name):
+ if not name or '$' not in name:
+ return name
+
+ return _expand_variables(name)
+
+
+def get_expanded_url(url):
+ url_path = url.split('://')
+ if len(url_path) != 2 or '$' not in url:
+ return url
+
+ return _expand_variables(url, '/')
+
+
+class YumUpdatePackageObject(object):
+
+ def __init__(self, name, arch, version, repo):
+ self.name = name
+ self.arch = arch
+ self.version = version
+ self.ui_from_repo = repo
+
+
+def _include_line_checkupdate_output(line):
+ tokens = line.split()
+
+ if len(tokens) != 3:
+ return False
+
+ if '.' not in tokens[0]:
+ return False
+
+ return True
+
+
+def _ignore_obsoleting_packages_in(output):
+ out = ''
+ for l in output.split('\n'):
+ if 'Obsoleting ' in l:
+ break
+ out += l + '\n'
+ return out
+
+
+def _filter_lines_checkupdate_output(output):
+ if output is None:
+ return []
+
+ output = _ignore_obsoleting_packages_in(output)
+
+ out = [l for l in output.split('\n')
+ if _include_line_checkupdate_output(l)]
+ return out
+
+
+def _get_yum_checkupdate_output():
+ cmd = ['yum', 'check-update', '-d0']
+ yum_update_cmd = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, error = yum_update_cmd.communicate()
+ return_code = yum_update_cmd.returncode
+ if return_code == 1:
+ return None
+
+ return out
+
+
+def get_yum_packages_list_update(checkupdate_output=None):
+ if checkupdate_output is None:
+ checkupdate_output = _get_yum_checkupdate_output()
+
+ filtered_output = _filter_lines_checkupdate_output(checkupdate_output)
+
+ packages = []
+ for line in filtered_output:
+ line = line.split()
+ index = 0
+ name_arch = line[index]
+ index += 1
+ version = line[index]
+ index += 1
+ repo = line[index]
+ name, arch = name_arch.rsplit('.', 1)
+ packages.append(YumUpdatePackageObject(name, arch, version, repo))
+
+ return packages
diff --git a/src/wok/plugins/kimchi/disks.py b/src/wok/plugins/kimchi/disks.py
deleted file mode 100644
index eb40e3a..0000000
--- a/src/wok/plugins/kimchi/disks.py
+++ /dev/null
@@ -1,196 +0,0 @@
-#
-# Project Kimchi
-#
-# Copyright IBM, Corp. 2013-2015
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-import os.path
-import re
-import subprocess
-from parted import Device as PDevice
-from parted import Disk as PDisk
-
-from wok.exception import OperationFailed
-from wok.utils import wok_log
-
-
-def _get_dev_node_path(maj_min):
- """ Returns device node path given the device number 'major:min' """
-
- dm_name = "/sys/dev/block/%s/dm/name" % maj_min
- if os.path.exists(dm_name):
- with open(dm_name) as dm_f:
- content = dm_f.read().rstrip('\n')
- return "/dev/mapper/" + content
-
- uevent = "/sys/dev/block/%s/uevent" % maj_min
- with open(uevent) as ueventf:
- content = ueventf.read()
-
- data = dict(re.findall(r'(\S+)=(".*?"|\S+)', content.replace("\n", " ")))
-
- return "/dev/%s" % data["DEVNAME"]
-
-
-def _get_lsblk_devs(keys, devs=[]):
- lsblk = subprocess.Popen(
- ["lsblk", "-Pbo"] + [','.join(keys)] + devs,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = lsblk.communicate()
- if lsblk.returncode != 0:
- raise OperationFailed("KCHDISKS0001E", {'err': err})
-
- return _parse_lsblk_output(out, keys)
-
-
-def _get_dev_major_min(name):
- maj_min = None
-
- keys = ["NAME", "MAJ:MIN"]
- dev_list = _get_lsblk_devs(keys)
-
- for dev in dev_list:
- if dev['name'].split()[0] == name:
- maj_min = dev['maj:min']
- break
- else:
- raise OperationFailed("KCHDISKS0002E", {'device': name})
-
- return maj_min
-
-
-def _is_dev_leaf(devNodePath):
- try:
- # By default, lsblk prints a device information followed by children
- # device information
- childrenCount = len(
- _get_lsblk_devs(["NAME"], [devNodePath])) - 1
- except OperationFailed as e:
- # lsblk is known to fail on multipath devices
- # Assume these devices contain children
- wok_log.error(
- "Error getting device info for %s: %s", devNodePath, e)
- return False
-
- return childrenCount == 0
-
-
-def _is_dev_extended_partition(devType, devNodePath):
- if devType != 'part':
- return False
- diskPath = devNodePath.rstrip('0123456789')
- device = PDevice(diskPath)
- try:
- extended_part = PDisk(device).getExtendedPartition()
- except NotImplementedError as e:
- wok_log.warning(
- "Error getting extended partition info for dev %s type %s: %s",
- devNodePath, devType, e.message)
- # Treate disk with unsupported partiton table as if it does not
- # contain extended partitions.
- return False
- if extended_part and extended_part.path == devNodePath:
- return True
- return False
-
-
-def _parse_lsblk_output(output, keys):
- # output is on format key="value",
- # where key can be NAME, TYPE, FSTYPE, SIZE, MOUNTPOINT, etc
- lines = output.rstrip("\n").split("\n")
- r = []
- for line in lines:
- d = {}
- for key in keys:
- expression = r"%s=\".*?\"" % key
- match = re.search(expression, line)
- field = match.group()
- k, v = field.split('=', 1)
- d[k.lower()] = v[1:-1]
- r.append(d)
- return r
-
-
-def _get_vgname(devNodePath):
- """ Return volume group name of a physical volume. If the device node path
- is not a physical volume, return empty string. """
- pvs = subprocess.Popen(
- ["pvs", "--unbuffered", "--nameprefixes", "--noheadings",
- "-o", "vg_name", devNodePath],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- out, err = pvs.communicate()
- if pvs.returncode != 0:
- return ""
-
- return re.findall(r"LVM2_VG_NAME='([^\']*)'", out)[0]
-
-
-def _is_available(name, devtype, fstype, mountpoint, majmin):
- devNodePath = _get_dev_node_path(majmin)
- # Only list unmounted and unformated and leaf and (partition or disk)
- # leaf means a partition, a disk has no partition, or a disk not held
- # by any multipath device. Physical volume belongs to no volume group
- # is also listed. Extended partitions should not be listed.
- if (devtype in ['part', 'disk', 'mpath'] and
- fstype in ['', 'LVM2_member'] and
- mountpoint == "" and
- _get_vgname(devNodePath) == "" and
- _is_dev_leaf(devNodePath) and
- not _is_dev_extended_partition(devtype, devNodePath)):
- return True
- return False
-
-
-def get_partitions_names(check=False):
- names = set()
- keys = ["NAME", "TYPE", "FSTYPE", "MOUNTPOINT", "MAJ:MIN"]
- # output is on format key="value",
- # where key can be NAME, TYPE, FSTYPE, MOUNTPOINT
- for dev in _get_lsblk_devs(keys):
- # split()[0] to avoid the second part of the name, after the
- # whiteline
- name = dev['name'].split()[0]
- if check and not _is_available(name, dev['type'], dev['fstype'],
- dev['mountpoint'], dev['maj:min']):
- continue
- names.add(name)
-
- return list(names)
-
-
-def get_partition_details(name):
- majmin = _get_dev_major_min(name)
- dev_path = _get_dev_node_path(majmin)
-
- keys = ["TYPE", "FSTYPE", "SIZE", "MOUNTPOINT"]
- try:
- dev = _get_lsblk_devs(keys, [dev_path])[0]
- except OperationFailed as e:
- wok_log.error(
- "Error getting partition info for %s: %s", name, e)
- return {}
-
- dev['available'] = _is_available(name, dev['type'], dev['fstype'],
- dev['mountpoint'], majmin)
- if dev['mountpoint']:
- # Sometimes the mountpoint comes with [SWAP] or other
- # info which is not an actual mount point. Filtering it
- regexp = re.compile(r"\[.*\]")
- if regexp.search(dev['mountpoint']) is not None:
- dev['mountpoint'] = ''
- dev['path'] = dev_path
- dev['name'] = name
- return dev
diff --git a/src/wok/plugins/kimchi/repositories.py b/src/wok/plugins/kimchi/repositories.py
deleted file mode 100644
index 2d6bac2..0000000
--- a/src/wok/plugins/kimchi/repositories.py
+++ /dev/null
@@ -1,534 +0,0 @@
-#
-# Project Kimchi
-#
-# Copyright IBM, Corp. 2014-2015
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-import copy
-import os
-import time
-import urlparse
-from ConfigParser import ConfigParser
-
-from wok.basemodel import Singleton
-from wok.exception import InvalidOperation, InvalidParameter
-from wok.exception import OperationFailed, NotFoundError, MissingParameter
-
-from wok.plugins.kimchi.config import kimchiLock
-from wok.plugins.kimchi.utils import validate_repo_url
-from wok.plugins.kimchi.yumparser import get_yum_repositories
-from wok.plugins.kimchi.yumparser import write_repo_to_file, get_display_name
-from wok.plugins.kimchi.yumparser import get_expanded_url
-
-
-class Repositories(object):
- __metaclass__ = Singleton
-
- """
- Class to represent and operate with repositories information.
- """
- def __init__(self):
- try:
- __import__('yum')
- self._pkg_mnger = YumRepo()
- except ImportError:
- try:
- __import__('apt_pkg')
- self._pkg_mnger = AptRepo()
- except ImportError:
- raise InvalidOperation('KCHREPOS0014E')
-
- def addRepository(self, params):
- """
- Add and enable a new repository
- """
- config = params.get('config', {})
- extra_keys = list(
- set(config.keys()).difference(set(self._pkg_mnger.CONFIG_ENTRY)))
- if len(extra_keys) > 0:
- raise InvalidParameter("KCHREPOS0028E",
- {'items': ",".join(extra_keys)})
-
- return self._pkg_mnger.addRepo(params)
-
- def getRepositories(self):
- """
- Return a dictionary with all Kimchi's repositories. Each element uses
- the format {<repo_id>: {repo}}, where repo is a dictionary in the
- repositories.Repositories() format.
- """
- return self._pkg_mnger.getRepositoriesList()
-
- def getRepository(self, repo_id):
- """
- Return a dictionary with all info from a given repository ID.
- """
- info = self._pkg_mnger.getRepo(repo_id)
- info['repo_id'] = repo_id
- return info
-
- def enableRepository(self, repo_id):
- """
- Enable a repository.
- """
- return self._pkg_mnger.toggleRepo(repo_id, True)
-
- def disableRepository(self, repo_id):
- """
- Disable a given repository.
- """
- return self._pkg_mnger.toggleRepo(repo_id, False)
-
- def updateRepository(self, repo_id, params):
- """
- Update the information of a given repository.
- The input is the repo_id of the repository to be updated and a dict
- with the information to be updated.
- """
- return self._pkg_mnger.updateRepo(repo_id, params)
-
- def removeRepository(self, repo_id):
- """
- Remove a given repository
- """
- return self._pkg_mnger.removeRepo(repo_id)
-
-
-class YumRepo(object):
- """
- Class to represent and operate with YUM repositories.
- It's loaded only on those systems listed at YUM_DISTROS and loads necessary
- modules in runtime.
- """
- TYPE = 'yum'
- DEFAULT_CONF_DIR = "/etc/yum.repos.d"
- CONFIG_ENTRY = ('repo_name', 'mirrorlist', 'metalink')
-
- def __init__(self):
- self._confdir = self.DEFAULT_CONF_DIR
-
- def _get_repos(self, errcode):
- try:
- kimchiLock.acquire()
- repos = get_yum_repositories()
- except Exception, e:
- kimchiLock.release()
- raise OperationFailed(errcode, {'err': str(e)})
- finally:
- kimchiLock.release()
-
- return repos
-
- def getRepositoriesList(self):
- """
- Return a list of repositories IDs
- """
- repos = self._get_repos('KCHREPOS0024E')
- return repos.keys()
-
- def getRepo(self, repo_id):
- """
- Return a dictionary in the repositories.Repositories() of the given
- repository ID format with the information of a YumRepository object.
- """
- repos = self._get_repos('KCHREPOS0025E')
-
- if repo_id not in repos.keys():
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- entry = repos.get(repo_id)
-
- display_name = get_display_name(entry.name)
-
- info = {}
- info['enabled'] = entry.enabled
- info['baseurl'] = entry.baseurl or ''
- info['config'] = {}
- info['config']['display_repo_name'] = display_name
- info['config']['repo_name'] = entry.name or ''
- info['config']['gpgcheck'] = entry.gpgcheck
- info['config']['gpgkey'] = entry.gpgkey or ''
- info['config']['mirrorlist'] = entry.mirrorlist or ''
- info['config']['metalink'] = entry.metalink or ''
- return info
-
- def addRepo(self, params):
- """
- Add a given repository to YumBase
- """
- # At least one base url, or one mirror, must be given.
- baseurl = params.get('baseurl', '')
-
- config = params.get('config', {})
- mirrorlist = config.get('mirrorlist', '')
- metalink = config.get('metalink', '')
- if not baseurl and not mirrorlist and not metalink:
- raise MissingParameter("KCHREPOS0013E")
-
- if baseurl:
- validate_repo_url(get_expanded_url(baseurl))
-
- if mirrorlist:
- validate_repo_url(get_expanded_url(mirrorlist))
-
- if metalink:
- validate_repo_url(get_expanded_url(metalink))
-
- if mirrorlist and metalink:
- raise InvalidOperation('KCHREPOS0030E')
-
- repo_id = params.get('repo_id', None)
- if repo_id is None:
- repo_id = "kimchi_repo_%s" % str(int(time.time() * 1000))
-
- repos = self._get_repos('KCHREPOS0026E')
- if repo_id in repos.keys():
- raise InvalidOperation("KCHREPOS0022E", {'repo_id': repo_id})
-
- repo_name = config.get('repo_name', repo_id)
- repo = {'baseurl': baseurl, 'mirrorlist': mirrorlist,
- 'name': repo_name, 'gpgcheck': 1,
- 'gpgkey': [], 'enabled': 1, 'metalink': metalink}
-
- # write a repo file in the system with repo{} information.
- parser = ConfigParser()
- parser.add_section(repo_id)
-
- for key, value in repo.iteritems():
- if value:
- parser.set(repo_id, key, value)
-
- repofile = os.path.join(self._confdir, repo_id + '.repo')
- try:
- with open(repofile, 'w') as fd:
- parser.write(fd)
- except:
- raise OperationFailed("KCHREPOS0018E",
- {'repo_file': repofile})
-
- return repo_id
-
- def toggleRepo(self, repo_id, enable):
- repos = self._get_repos('KCHREPOS0011E')
- if repo_id not in repos.keys():
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- entry = repos.get(repo_id)
- if enable and entry.enabled:
- raise InvalidOperation("KCHREPOS0015E", {'repo_id': repo_id})
-
- if not enable and not entry.enabled:
- raise InvalidOperation("KCHREPOS0016E", {'repo_id': repo_id})
-
- kimchiLock.acquire()
- try:
- if enable:
- entry.enable()
- else:
- entry.disable()
-
- write_repo_to_file(entry)
- except:
- if enable:
- raise OperationFailed("KCHREPOS0020E", {'repo_id': repo_id})
-
- raise OperationFailed("KCHREPOS0021E", {'repo_id': repo_id})
- finally:
- kimchiLock.release()
-
- return repo_id
-
- def updateRepo(self, repo_id, params):
- """
- Update a given repository in repositories.Repositories() format
- """
- repos = self._get_repos('KCHREPOS0011E')
- if repo_id not in repos.keys():
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- entry = repos.get(repo_id)
-
- baseurl = params.get('baseurl', None)
- config = params.get('config', {})
- mirrorlist = config.get('mirrorlist', None)
- metalink = config.get('metalink', None)
-
- if baseurl is not None and len(baseurl.strip()) == 0:
- baseurl = None
-
- if mirrorlist is not None and len(mirrorlist.strip()) == 0:
- mirrorlist = None
-
- if metalink is not None and len(metalink.strip()) == 0:
- metalink = None
-
- if baseurl is None and mirrorlist is None and metalink is None:
- raise MissingParameter("KCHREPOS0013E")
-
- if baseurl is not None:
- validate_repo_url(get_expanded_url(baseurl))
- entry.baseurl = baseurl
-
- if mirrorlist is not None:
- validate_repo_url(get_expanded_url(mirrorlist))
- entry.mirrorlist = mirrorlist
-
- if metalink is not None:
- validate_repo_url(get_expanded_url(metalink))
- entry.metalink = metalink
-
- if mirrorlist and metalink:
- raise InvalidOperation('KCHREPOS0030E')
-
- entry.id = params.get('repo_id', repo_id)
- entry.name = config.get('repo_name', entry.name)
- entry.gpgcheck = config.get('gpgcheck', entry.gpgcheck)
- entry.gpgkey = config.get('gpgkey', entry.gpgkey)
- kimchiLock.acquire()
- write_repo_to_file(entry)
- kimchiLock.release()
- return repo_id
-
- def removeRepo(self, repo_id):
- """
- Remove a given repository
- """
- repos = self._get_repos('KCHREPOS0027E')
- if repo_id not in repos.keys():
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- entry = repos.get(repo_id)
- parser = ConfigParser()
- with open(entry.repofile) as fd:
- parser.readfp(fd)
-
- if len(parser.sections()) == 1:
- os.remove(entry.repofile)
- return
-
- parser.remove_section(repo_id)
- with open(entry.repofile, "w") as fd:
- parser.write(fd)
-
-
-class AptRepo(object):
- """
- Class to represent and operate with YUM repositories.
- It's loaded only on those systems listed at YUM_DISTROS and loads necessary
- modules in runtime.
- """
- TYPE = 'deb'
- KIMCHI_LIST = "kimchi-source.list"
- CONFIG_ENTRY = ('dist', 'comps')
-
- def __init__(self):
- getattr(__import__('apt_pkg'), 'init_config')()
- getattr(__import__('apt_pkg'), 'init_system')()
- config = getattr(__import__('apt_pkg'), 'config')
- self.pkg_lock = getattr(__import__('apt_pkg'), 'SystemLock')
- module = __import__('aptsources.sourceslist', globals(), locals(),
- ['SourcesList'], -1)
-
- self._sourceparts_path = '/%s%s' % (
- config.get('Dir::Etc'), config.get('Dir::Etc::sourceparts'))
- self._sourceslist = getattr(module, 'SourcesList')
- self.filename = os.path.join(self._sourceparts_path, self.KIMCHI_LIST)
- if not os.path.exists(self.filename):
- with open(self.filename, 'w') as fd:
- fd.write("# This file is managed by Kimchi and it must not "
- "be modified manually\n")
-
- def _get_repos(self):
- try:
- with self.pkg_lock():
- repos = self._sourceslist()
- repos.refresh()
- except Exception, e:
- kimchiLock.release()
- raise OperationFailed('KCHREPOS0025E', {'err': e.message})
-
- return repos
-
- def _get_repo_id(self, repo):
- data = urlparse.urlparse(repo.uri)
- name = data.hostname or data.path
- return '%s-%s-%s' % (name, repo.dist, "-".join(repo.comps))
-
- def _get_source_entry(self, repo_id):
- kimchiLock.acquire()
- repos = self._get_repos()
- kimchiLock.release()
-
- for r in repos:
- # Ignore deb-src repositories
- if r.type != 'deb':
- continue
-
- if self._get_repo_id(r) != repo_id:
- continue
-
- return r
-
- return None
-
- def getRepositoriesList(self):
- """
- Return a list of repositories IDs
-
- APT repositories there aren't the concept about repository ID, so for
- internal control, the repository ID will be built as described in
- _get_repo_id()
- """
- kimchiLock.acquire()
- repos = self._get_repos()
- kimchiLock.release()
-
- res = []
- for r in repos:
- # Ignore deb-src repositories
- if r.type != 'deb':
- continue
-
- res.append(self._get_repo_id(r))
-
- return res
-
- def getRepo(self, repo_id):
- """
- Return a dictionary in the repositories.Repositories() format of the
- given repository ID with the information of a SourceEntry object.
- """
- r = self._get_source_entry(repo_id)
- if r is None:
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- info = {'enabled': not r.disabled,
- 'baseurl': r.uri,
- 'config': {'dist': r.dist,
- 'comps': r.comps}}
- return info
-
- def addRepo(self, params):
- """
- Add a new APT repository based on <params>
- """
- # To create a APT repository the dist is a required parameter
- # (in addition to baseurl, verified on controller through API.json)
- config = params.get('config', None)
- if config is None:
- raise MissingParameter("KCHREPOS0019E")
-
- if 'dist' not in config.keys():
- raise MissingParameter("KCHREPOS0019E")
-
- uri = params['baseurl']
- dist = config['dist']
- comps = config.get('comps', [])
-
- validate_repo_url(get_expanded_url(uri))
-
- kimchiLock.acquire()
- try:
- repos = self._get_repos()
- source_entry = repos.add('deb', uri, dist, comps,
- file=self.filename)
- with self.pkg_lock():
- repos.save()
- except Exception as e:
- kimchiLock.release()
- raise OperationFailed("KCHREPOS0026E", {'err': e.message})
- kimchiLock.release()
- return self._get_repo_id(source_entry)
-
- def toggleRepo(self, repo_id, enable):
- """
- Enable a given repository
- """
- r = self._get_source_entry(repo_id)
- if r is None:
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- if enable and not r.disabled:
- raise InvalidOperation("KCHREPOS0015E", {'repo_id': repo_id})
-
- if not enable and r.disabled:
- raise InvalidOperation("KCHREPOS0016E", {'repo_id': repo_id})
-
- if enable:
- line = 'deb'
- else:
- line = '#deb'
-
- kimchiLock.acquire()
- try:
- repos = self._get_repos()
- with self.pkg_lock():
- repos.remove(r)
- repos.add(line, r.uri, r.dist, r.comps, file=self.filename)
- repos.save()
- except:
- kimchiLock.release()
- if enable:
- raise OperationFailed("KCHREPOS0020E", {'repo_id': repo_id})
-
- raise OperationFailed("KCHREPOS0021E", {'repo_id': repo_id})
- finally:
- kimchiLock.release()
-
- return repo_id
-
- def updateRepo(self, repo_id, params):
- """
- Update a given repository in repositories.Repositories() format
- """
- old_info = self.getRepo(repo_id)
- updated_info = copy.deepcopy(old_info)
- updated_info['baseurl'] = params.get(
- 'baseurl', updated_info['baseurl'])
-
- if 'config' in params.keys():
- config = params['config']
- updated_info['config']['dist'] = config.get(
- 'dist', old_info['config']['dist'])
- updated_info['config']['comps'] = config.get(
- 'comps', old_info['config']['comps'])
-
- self.removeRepo(repo_id)
- try:
- return self.addRepo(updated_info)
- except:
- self.addRepo(old_info)
- raise
-
- def removeRepo(self, repo_id):
- """
- Remove a given repository
- """
- r = self._get_source_entry(repo_id)
- if r is None:
- raise NotFoundError("KCHREPOS0012E", {'repo_id': repo_id})
-
- kimchiLock.acquire()
- try:
- repos = self._get_repos()
- with self.pkg_lock():
- repos.remove(r)
- repos.save()
- except:
- kimchiLock.release()
- raise OperationFailed("KCHREPOS0017E", {'repo_id': repo_id})
- finally:
- kimchiLock.release()
diff --git a/src/wok/plugins/kimchi/yumparser.py b/src/wok/plugins/kimchi/yumparser.py
deleted file mode 100644
index a481ac2..0000000
--- a/src/wok/plugins/kimchi/yumparser.py
+++ /dev/null
@@ -1,353 +0,0 @@
-#
-# Project Kimchi
-#
-# Copyright IBM, Corp. 2015
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-import subprocess
-import glob
-
-from os import listdir
-from os.path import isfile, splitext, basename
-
-try:
- import rpm
-except ImportError:
- pass
-
-
-class YumRepoObject(object):
-
- def __init__(self, repo_id, repofile):
- self.repo_id = repo_id
- self.name = None
- self.baseurl = None
- self.enabled = True
- self.gpgcheck = True
- self.gpgkey = None
- self.metalink = None
- self.mirrorlist = None
- self.repofile = repofile
- self.string_attrs = ['baseurl', 'gpgkey', 'name',
- 'metalink', 'mirrorlist']
- self.boolean_attrs = ['enabled', 'gpgcheck']
-
- def set_attribute(self, key, strvalue):
- if key in self.string_attrs:
- setattr(self, key, strvalue)
- elif key in self.boolean_attrs:
- setattr(self, key, (strvalue == '1'))
-
- def get_attribute_str(self, key):
- if key not in self.get_attributes():
- return None
-
- if key in self.boolean_attrs:
- str_value = '1' if getattr(self, key) is True else '0'
- else:
- str_value = getattr(self, key)
-
- if str_value is None:
- return None
-
- return key + '=' + str_value
-
- def get_attributes(self):
- return self.string_attrs + self.boolean_attrs
-
- def enable(self):
- self.enabled = True
-
- def disable(self):
- self.enabled = False
-
- def __str__(self):
- str_obj = '[' + self.repo_id + ']' + '\n'
- for key in self.get_attributes():
- if self.get_attribute_str(key) is not None:
- str_obj += self.get_attribute_str(key) + '\n'
- return str_obj
-
-
-def get_repo_files():
- def _is_repository_file(f):
- _, f_extension = splitext(f)
- return isfile(f) and (f_extension == '.repo')
-
- YUM_REPO_DIR = '/etc/yum.repos.d'
- return [YUM_REPO_DIR+'/'+f for f in listdir(YUM_REPO_DIR)
- if _is_repository_file(YUM_REPO_DIR+'/'+f)]
-
-
-def _ignore_line_repo_file(line):
- return line.startswith("#") or '=' not in line
-
-
-def _get_repos_from_file(repo_file):
- repos_from_file = {}
- current_repo = None
- current_repo_id = None
- with open(repo_file) as f:
- for line in f.readlines():
- line = line.strip()
- if line.startswith("["):
- if current_repo is not None:
- repos_from_file[current_repo_id] = current_repo
- current_repo_id = line.strip('[]')
- current_repo = YumRepoObject(current_repo_id, repo_file)
- continue
- if _ignore_line_repo_file(line):
- continue
- key, value = line.split('=', 1)
- key = key.strip()
- value = value.strip()
- current_repo.set_attribute(key, value)
-
- # add the last repo from file.
- if current_repo is not None:
- repos_from_file[current_repo_id] = current_repo
-
- return repos_from_file
-
-
-def get_yum_repositories():
- repo_files = get_repo_files()
- repos = {}
- for yum_repo in repo_files:
- repos.update(_get_repos_from_file(yum_repo))
-
- return repos
-
-
-def _retrieve_repo_line_index(data, repo):
- repo_entry = '[' + repo.repo_id + ']\n'
- try:
- repo_index = data.index(repo_entry)
- except:
- return None
- return repo_index
-
-
-def _update_repo_file_data(data, repo, repo_index):
- remaining_repo_attrs = repo.get_attributes()
-
- for i in range(repo_index + 1, len(data)):
- line = data[i].strip()
- if line.startswith('['):
- break
- if _ignore_line_repo_file(line):
- continue
- key, _ = line.split('=', 1)
- key = key.strip()
- attr_str = repo.get_attribute_str(key)
- if attr_str is None:
- continue
- remaining_repo_attrs.remove(key)
- data[i] = attr_str + '\n'
-
- for attr in remaining_repo_attrs:
- attr_str = repo.get_attribute_str(attr)
- if attr_str is None:
- continue
- data.insert(repo_index+1, attr_str + '\n')
-
- return data
-
-
-def write_repo_to_file(repo):
- with open(repo.repofile) as f:
- data = f.readlines()
-
- repo_index = _retrieve_repo_line_index(data, repo)
- if repo_index is None:
- return
-
- data = _update_repo_file_data(data, repo, repo_index)
-
- with open(repo.repofile, 'w') as f:
- f.writelines(data)
-
-
-def _get_last_line_repo(data, repo_index):
- stop_delete_index = None
- for i in range(repo_index+1, len(data)):
- line = data[i].strip()
- if line.startswith('['):
- stop_delete_index = i - 1
- break
- if stop_delete_index is None:
- stop_delete_index = len(data) - 1
-
- return stop_delete_index
-
-
-def _remove_repo_file_data(data, repo_index):
- last_line_repo = _get_last_line_repo(data, repo_index)
- for i in range(last_line_repo, repo_index - 1, -1):
- data.pop(i)
- return data
-
-
-def delete_repo_from_file(repo):
- with open(repo.repofile) as f:
- data = f.readlines()
-
- repo_index = _retrieve_repo_line_index(data, repo)
- if repo_index is None:
- return
-
- data = _remove_repo_file_data(data, repo_index)
-
- with open(repo.repofile, 'w') as f:
- f.writelines(data)
-
-
-def _get_releasever():
- release_file = glob.glob('/etc/*-release')[0]
- transaction = rpm.TransactionSet()
- match_iter = transaction.dbMatch('basenames', release_file)
-
- ret = '%releasever'
- try:
- ret = match_iter.next()['version']
-
- except StopIteration:
- pass
-
- return ret
-
-
-def _get_basearch():
- cmd = ['uname', '-i']
- uname = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- return uname.communicate()[0].strip('"\n')
-
-
-def _get_all_yum_vars():
- variables = {}
-
- def _get_var_content(varfile):
- with open(varfile) as f:
- variables[basename(varfile)] = f.read().strip('\n')
-
- map(lambda vfile:
- _get_var_content(vfile),
- glob.glob('/etc/yum/vars/*'))
-
- return variables
-
-
-def _expand_variables(stringvar, split_char=' '):
- yum_variables = _get_all_yum_vars()
- yum_variables['releasever'] = _get_releasever()
- yum_variables['basearch'] = _get_basearch()
-
- name_vars = [var for var in stringvar.split(split_char)
- if var.startswith('$') and var.strip('$') in yum_variables]
-
- return reduce(lambda nm, var:
- nm.replace(var, yum_variables[var.strip('$')]),
- name_vars,
- stringvar)
-
-
-def get_display_name(name):
- if not name or '$' not in name:
- return name
-
- return _expand_variables(name)
-
-
-def get_expanded_url(url):
- url_path = url.split('://')
- if len(url_path) != 2 or '$' not in url:
- return url
-
- return _expand_variables(url, '/')
-
-
-class YumUpdatePackageObject(object):
-
- def __init__(self, name, arch, version, repo):
- self.name = name
- self.arch = arch
- self.version = version
- self.ui_from_repo = repo
-
-
-def _include_line_checkupdate_output(line):
- tokens = line.split()
-
- if len(tokens) != 3:
- return False
-
- if '.' not in tokens[0]:
- return False
-
- return True
-
-
-def _ignore_obsoleting_packages_in(output):
- out = ''
- for l in output.split('\n'):
- if 'Obsoleting ' in l:
- break
- out += l + '\n'
- return out
-
-
-def _filter_lines_checkupdate_output(output):
- if output is None:
- return []
-
- output = _ignore_obsoleting_packages_in(output)
-
- out = [l for l in output.split('\n')
- if _include_line_checkupdate_output(l)]
- return out
-
-
-def _get_yum_checkupdate_output():
- cmd = ['yum', 'check-update', '-d0']
- yum_update_cmd = subprocess.Popen(cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, error = yum_update_cmd.communicate()
- return_code = yum_update_cmd.returncode
- if return_code == 1:
- return None
-
- return out
-
-
-def get_yum_packages_list_update(checkupdate_output=None):
- if checkupdate_output is None:
- checkupdate_output = _get_yum_checkupdate_output()
-
- filtered_output = _filter_lines_checkupdate_output(checkupdate_output)
-
- packages = []
- for line in filtered_output:
- line = line.split()
- index = 0
- name_arch = line[index]
- index += 1
- version = line[index]
- index += 1
- repo = line[index]
- name, arch = name_arch.rsplit('.', 1)
- packages.append(YumUpdatePackageObject(name, arch, version, repo))
-
- return packages
--
2.1.0
More information about the Kimchi-devel
mailing list