[Kimchi-devel] [PATCH 3/5] Host's repositories management: Update backend.

Aline Manera alinefm at linux.vnet.ibm.com
Mon Feb 10 18:36:35 UTC 2014


On 02/10/2014 11:46 AM, Paulo Vital wrote:
> Update model and mockmodel to support backend opertions.
> Add new file implementing backend operations. There are two new classes:
>
> 1) Repositories (object): Class to represent and operate with repositories
> information in Kimchi's perspective. It's agnostic to host;s package management
> system, and can execute all operations necessary: add repository, get all
> repositories list, get information about one repository, update a repository,
> enable and disable a repository and remove a repository. This class will load
> in runtime the necessary classes to work with the host's package management:
> YumRepo for YUM systems based and AptRepo for APT systems based (support for
> this last one will be submited in a close future);
>
> 2) YumRepo (object): Class to represent and operate with YUM repositories.
> Loaded only on those systems that supports YUM, it's responsible to connect,
> collect and provide information of YUM repositories in the system. Also it's
> responsible to create/delete the files in disk to maintain the repositories in
> system after disconnection.
>
> Signed-off-by: Paulo Vital <pvital at linux.vnet.ibm.com>
> ---
>   src/kimchi/mockmodel.py    |  34 ++++
>   src/kimchi/model/host.py   |  50 +++++-
>   src/kimchi/repositories.py | 418 +++++++++++++++++++++++++++++++++++++++++++++
>   3 files changed, 501 insertions(+), 1 deletion(-)
>   create mode 100644 src/kimchi/repositories.py
>
> diff --git a/src/kimchi/mockmodel.py b/src/kimchi/mockmodel.py
> index 4e276eb..1ae97a2 100644
> --- a/src/kimchi/mockmodel.py
> +++ b/src/kimchi/mockmodel.py
> @@ -51,6 +51,7 @@ from kimchi.model.storagepools import ISO_POOL_NAME, STORAGE_SOURCES
>   from kimchi.model.utils import get_vm_name
>   from kimchi.model.vms import VM_STATIC_UPDATE_PARAMS
>   from kimchi.objectstore import ObjectStore
> +from kimchi.repositories import Repositories
>   from kimchi.screenshot import VMScreenshot
>   from kimchi.utils import template_name_from_uri, pool_name_from_uri
>   from kimchi.vmtemplate import VMTemplate
> @@ -77,6 +78,7 @@ class MockModel(object):
>           self._mock_interfaces = self.dummy_interfaces()
>           self.next_taskid = 1
>           self.storagepool_activate('default')
> +        self.host_repositories = Repositories()
>
>       def _static_vm_update(self, dom, params):
>           state = dom.info['state']
> @@ -665,6 +667,38 @@ class MockModel(object):
>                   'display_proxy_port':
>                   kconfig.get('display', 'display_proxy_port')}
>
> +    def repositories_get_list(self):
> +        return [repo_id for repo_id in
> +                self.host_repositories.getRepositories().keys()]
> +

You should return default values here to be able to add tests for this 
feature.
As you did for update feature.

> +    def repositories_create(self, params):
> +        repo_id = params.get('repo_id')

If the repo_id was not provided by the user you should create one for him

> +        if repo_id in self.repositories_get_list():
> +            raise InvalidOperation("Repository %s already exists." % repo_id)
> +        self.host_repositories.addRepository(params)
> +        return repo_id
> +
> +    def repository_lookup(self, repo_id):
> +        return self.host_repositories.getRepository(repo_id)
> +
> +    def repository_delete(self, repo_id):
> +        return self.host_repositories.removeRepository(repo_id)
> +
> +    def repository_enable(self, repo_id):
> +        if not self.host_repositories.enableRepository(repo_id):
> +            raise OperationFailed("Could not enable repository %s." % repo_id)
> +
> +    def repository_disable(self, repo_id):
> +        if not  self.host_repositories.disableRepository(repo_id):
> +            raise OperationFailed("Could not disable repository %s." % repo_id)
> +
> +    def repository_update(self, repo_id, params):
> +        try:
> +            self.host_repositories.updateRepository(repo_id, params)
> +        except:
> +            raise OperationFailed("Could not update repository %s." % repo_id)
> +        return repo_id
> +
>
>   class MockVMTemplate(VMTemplate):
>       def __init__(self, args, mockmodel_inst=None):
> diff --git a/src/kimchi/model/host.py b/src/kimchi/model/host.py
> index a3d9e38..9b8b250 100644
> --- a/src/kimchi/model/host.py
> +++ b/src/kimchi/model/host.py
> @@ -31,8 +31,9 @@ from cherrypy.process.plugins import BackgroundTask
>   from kimchi import disks
>   from kimchi import netinfo
>   from kimchi.basemodel import Singleton
> -from kimchi.exception import NotFoundError, OperationFailed
> +from kimchi.exception import InvalidOperation, NotFoundError, OperationFailed
>   from kimchi.model.vms import DOM_STATE_MAP
> +from kimchi.repositories import Repositories
>   from kimchi.utils import kimchi_log
>
>
> @@ -199,3 +200,50 @@ class PartitionModel(object):
>               raise NotFoundError("Partition %s not found in the host"
>                                   % name)
>           return disks.get_partition_details(name)
> +
> +
> +class RepositoriesModel(object):
> +    __metaclass__ = Singleton
> +
> +    def __init__(self, **kargs):
> +        self.host_repositories = Repositories()
> +
> +    def get_list(self):
> +        return [repo_id for repo_id in
> +                self.host_repositories.getRepositories().keys()]
> +
> +    def create(self, params):
> +        repo_id = params.get('repo_id')

Same here.
You need to create a repo_id if user does not provide one

> +        if repo_id in self.get_list():
> +            raise InvalidOperation("Repository %s already exists." % repo_id)
> +        self.host_repositories.addRepository(params)
> +        return repo_id
> +
> +    def get_repositories(self):
> +        return self.host_repositories
> +
> +
> +class RepositoryModel(object):
> +    def __init__(self, **kargs):
> +        self._repositories = RepositoriesModel().get_repositories()
> +
> +    def lookup(self, repo_id):
> +        return self._repositories.getRepository(repo_id)
> +
> +    def enable(self, repo_id):
> +        if not self._repositories.enableRepository(repo_id):
> +            raise OperationFailed("Could not enable repository %s." % repo_id)
> +
> +    def disable(self, repo_id):
> +        if not self._repositories.disableRepository(repo_id):
> +            raise OperationFailed("Could not disable repository %s." % repo_id)
> +
> +    def update(self, repo_id, params):
> +        try:
> +            self._repositories.updateRepository(repo_id, params)
> +        except:
> +            raise OperationFailed("Could not update repository %s." % repo_id)
> +        return repo_id
> +
> +    def delete(self, repo_id):
> +        return self._repositories.removeRepository(repo_id)
> diff --git a/src/kimchi/repositories.py b/src/kimchi/repositories.py
> new file mode 100644
> index 0000000..f2b5105
> --- /dev/null
> +++ b/src/kimchi/repositories.py
> @@ -0,0 +1,418 @@
> +#
> +# Project Kimchi
> +#
> +# Copyright IBM, Corp. 2014
> +#
> +# Authors:
> +#  Paulo Vital <pvital at linux.vnet.ibm.com>
> +#
> +# This library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +#
> +# This library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +#
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with this library; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
> +
> +import os
> +import platform
> +
> +from kimchi.exception import InvalidParameter, OperationFailed, NotFoundError
> +
> +YUM_DISTROS = ['fedora', 'red hat enterprise linux',
> +               'red hat enterprise linux server', 'opensuse ',
> +               'suse linux enterprise server ']
> +APT_DISTROS = ['debian', 'ubuntu']
> +
> +
> +class Repositories (object):
> +    """
> +    Class to represent and operate with repositories information.
> +    """
> +    def __init__(self):
> +        # This stores all repositories for Kimchi perspective. It's a
> +        # dictionary of dictionaries, in the format {<repo_id>: {repo}},
> +        # where:
> +        # repo = {'repo_id': <string>, 'repo_name': <string>,
> +        #        'baseurl': ([<string>], None), 'url_args': ([<string>, None),
> +        #        'enabled': True/False, 'gpgcheck': True/False,
> +        #        'gpgkey': ([<string>], None),
> +        #        'mirrors': ([<string>], None)
> +        #       }


 From API.md we have:

* repo_id:...
* repo_name:...
* baseurl:...
* url_args: ...
* enabled: ...
* gpgcheck: ...
* gpgkey: ...

So you have return the same keys

> +        self._repo_storage = {}
> +
> +        self._distro = platform.linux_distribution()[0].lower()
> +        if (self._distro in YUM_DISTROS):
> +            self._pkg_mnger = YumRepo()
> +        elif (self._distro in APT_DISTROS):
> +            self._pkg_mnger = AptRepo()
> +        else:

> +            self._pkg_mnger = None

We need to raise an error is this case.


> +
> +        if self._pkg_mnger:
> +            # update the self._repo_storage with system's repositories
> +            self._scanSystemRepositories()
> +
> +    def _scanSystemRepositories(self):
> +        """
> +        Update repositories._repo_storage with system's (host) repositories.
> +        """
> +        # Call system pkg_mnger to get the repositories as list of dict.
> +        for repo in self._pkg_mnger.getRepositoriesList():
> +            self.addRepository(repo)
> +
> +    def addRepository(self, params={}):
> +        """
> +        Add and enable a new repository into repositories._repo_storage.
> +        """
> +        if not params:
> +            raise InvalidParameter("No parameters to add repository.")

MissingParameter()

> +
> +        # Check if already exist a repository with the same repo_id.
> +        repo_id = params.get('repo_id')
> +        if repo_id in self.getRepositories():
> +            raise OperationFailed("Repository %s already exists." % repo_id)

InvalidParameter()

> +
> +        # Create and enable the repository
> +        repo = {'repo_id': repo_id,
> +                'repo_name': params.get('repo_name', repo_id),
> +                'baseurl': params.get('baseurl', None),
> +                'url_args': params.get('url_args', None),
> +                'enabled': True,
> +                'gpgkey': params.get('gpgkey', None),
> +                'mirrors': params.get('mirrors', None)}
> +
> +        if repo['gpgkey'] != None:
> +            repo['gpgcheck'] = True
> +        else:
> +            repo['gpgcheck'] = False
> +
> +        self._repo_storage[repo_id] = repo
> +
> +        # Check in self._pkg_mnger if the repository already exists there
> +        if not repo_id in [irepo['repo_id'] for irepo in
> +                           self._pkg_mnger.getRepositoriesList()]:
> +            self._pkg_mnger.addRepo(repo)
> +
> +    def getRepositories(self):
> +        """
> +        Return a dictionary with all Kimchi's repositories. Each element uses
> +        the format {<repo_id>: {repo}}, where repo is a dictionary in the
> +        repositories.Repositories() format.
> +        """
> +        return self._repo_storage
> +
> +    def getRepository(self, repo_id):
> +        """
> +        Return a dictionary with all info from a given repository ID.
> +        """
> +        if not repo_id in self._repo_storage.keys():
> +            raise NotFoundError("Repository %s does not exists." % repo_id)
> +
> +        repo = self._repo_storage[repo_id]
> +        if (isinstance(repo['baseurl'], list)) and (len(repo['baseurl']) > 0):
> +            repo['baseurl'] = repo['baseurl'][0]
> +        if isinstance(repo['mirrors'], list) and (len(repo['mirrors']) > 0):
> +            repo['mirrors'] = repo['mirrors'][0]
> +

 From API.md we have:

* repo_id:...
* repo_name:...
* baseurl:...
* url_args: ...
* enabled: ...
* gpgcheck: ...
* gpgkey: ...

So you have return the same keys

> +        return repo
> +
> +    def getRepositoryFromPkgMnger(self, repo_id):
> +        """
> +        Return a dictionary with all info from a given repository ID.
> +        All info come from self._pkg_mnger.getRepo().
> +        """
> +        return self._pkg_mnger.getRepo(repo_id)
> +
> +    def enabledRepositories(self):
> +        """
> +        Return a list with enabled repositories IDs.
> +        """
> +        enabled_repos = []
> +        for repo_id in self._repo_storage.keys():
> +            if self._repo_storage[repo_id]['enabled']:
> +                enabled_repos.append(repo_id)
> +        return enabled_repos
> +
> +    def enableRepository(self, repo_id):
> +        """
> +        Enable a repository.
> +        """
> +        # Check if repo_id is already enabled
> +        if repo_id in self.enabledRepositories():
> +            raise NotFoundError("There is no disabled repository called %s." %
> +                                repo_id)
> +
> +        try:
> +            repo = self.getRepository(repo_id)
> +            repo['enabled'] = True
> +            self.updateRepository(repo_id, repo)
> +            self._pkg_mnger.enableRepo(repo_id)
> +            return True
> +        except:
> +            raise OperationFailed("Could not enable repository %s" % repo_id)
> +
> +    def disableRepository(self, repo_id):
> +        """
> +        Disable a given repository.
> +        """
> +        # Check if repo_id is already disabled
> +        if not repo_id in self.enabledRepositories():
> +            raise NotFoundError("There is no enabled repository called %s." %
> +                                repo_id)
> +
> +        try:
> +            repo = self.getRepository(repo_id)
> +            repo['enabled'] = False
> +            self.updateRepository(repo_id, repo)
> +            self._pkg_mnger.disableRepo(repo_id)
> +            return True
> +        except:
> +            raise OperationFailed("Could not disable repository %s" % repo_id)
> +
> +    def updateRepository(self, repo_id, new_repo={}):
> +        """
> +        Update the information of a given repository.
> +        The input is the repo_id of the repository to be updated and a dict
> +        with the information to be updated.
> +        """
> +        if (len(new_repo) == 0):
> +            raise InvalidParameter("No parameters to update repository.")
> +
> +        repo = self._repo_storage[repo_id]
> +        repo.update(new_repo)
> +
> +        del self._repo_storage[repo_id]
> +        self._repo_storage[repo_id] = repo
> +        self._pkg_mnger.updateRepo(repo_id, self._repo_storage[repo_id])
> +
> +    def removeRepository(self, repo_id):
> +        """
> +        Remove a given repository
> +        """
> +        if not repo_id in self._repo_storage.keys():
> +            raise NotFoundError("There is no repository called %s." % repo_id)
> +
> +        del self._repo_storage[repo_id]
> +        self._pkg_mnger.removeRepo(repo_id)
> +        return True
> +
> +
> +class YumRepo (object):
> +    """
> +    Class to represent and operate with YUM repositories.
> +    It's loaded only on those systems listed at YUM_DISTROS and loads necessary
> +    modules in runtime.
> +    """
> +    def __init__(self):
> +        self._yb = getattr(__import__('yum'), 'YumBase')()
> +        self._repos = self._yb.repos
> +        self._conf = self._yb.conf
> +        self._enabled_repos = self._repos.listEnabled()
> +
> +    def getRepositoriesList(self):
> +        """
> +        Return a list of dictionaries in the repositories.Repositories() format
> +        """
> +        repo_list = []
> +        for repo in self.enabledRepos():
> +            irepo = {}
> +            irepo['repo_id'] = repo.id
> +            irepo['repo_name'] = repo.name
> +            irepo['baseurl'] = repo.baseurl
> +            irepo['mirrors'] = repo.mirrorlist
> +            irepo['url_args'] = None,
> +            irepo['enabled'] = repo.enabled
> +            irepo['gpgcheck'] = repo.gpgcheck
> +            irepo['gpgkey'] = repo.gpgkey
> +            repo_list.append(irepo)
> +        return repo_list
> +
> +    def addRepo(self, repo={}):
> +        """
> +        Add a given repository in repositories.Repositories() format to YumBase
> +        """
> +        if len(repo) == 0:
> +            raise InvalidParameter("No parameters to add repository.")
> +
> +        # At least one base url, or one mirror, must be given.
> +        # baseurls must be a list of strings specifying the urls
> +        # mirrorlist must be a list of strings specifying a list of mirrors
> +        # Here we creates the lists, or set as None
> +        if not repo['baseurl']:
> +            baseurl = []

The baseurl is a required parameter. Must always be there.

> +        else:
> +            baseurl = [repo['baseurl']]
> +

> +        if not repo['mirrors']:
> +            mirrors = None
> +        else:
> +            mirrors = repo['mirrors']

 From API.md it should be 'is_mirror' that indicates if the 'baseurl' is 
a mirror or not.

> +
> +        self._yb.add_enable_repo(repo['repo_id'], baseurl, mirrors,
> +                                 name=repo['repo_name'],
> +                                 gpgcheck=repo['gpgcheck'],
> +                                 gpgkey=[repo['gpgkey']])
> +
> +        repo['baseurl'] = baseurl
> +        repo['mirrors'] = mirrors
> +        # write a repo file in the system with repo{} information.
> +        self._write2disk(repo)
> +
> +    def getRepo(self, repo_id):
> +        """
> +        Return a dictionary in the repositories.Repositories() of the given
> +        repository ID format with the information of a YumRepository object.
> +        """
> +        try:
> +            repo = self._repos.getRepo(repo_id)
> +            irepo = {}
> +            irepo['repo_id'] = repo.id
> +            irepo['repo_name'] = repo.name
> +            irepo['baseurl'] = repo.baseurl
> +            irepo['mirrors'] = repo.mirrorlist
> +            irepo['url_args'] = None,
> +            irepo['enabled'] = repo.enabled
> +            irepo['gpgcheck'] = repo.gpgcheck
> +            irepo['gpgkey'] = repo.gpgkey
> +            return irepo
> +        except:
> +            raise OperationFailed("Repository %s does not exists." % repo_id)
> +
> +    def enabledRepos(self):
> +        """
> +        Return a list with enabled YUM repositories IDs
> +        """
> +        return self._enabled_repos
> +
> +    def isRepoEnable(self, repo_id):
> +        """
> +        Return if a given repository ID is enabled or not
> +        """
> +        for repo in self.enabledRepos():
> +            if repo_id == repo.id:
> +                return True
> +        return False
> +
> +    def enableRepo(self, repo_id):
> +        """
> +        Enable a given repository
> +        """
> +        try:
> +            self._repos.getRepo(repo_id).enable()
> +            self._repos.doSetup()
> +            return True
> +        except:
> +            raise OperationFailed("Could not enable repository %s." % repo_id)
> +
> +    def disableRepo(self, repo_id):
> +        """
> +        Disable a given repository
> +        """
> +        try:
> +            self._repos.getRepo(repo_id).disable()
> +            self._repos.doSetup()
> +            return True
> +        except:
> +            raise OperationFailed("Could not disable repository %s." % repo_id)
> +
> +    def updateRepo(self, repo_id, repo={}):
> +        """
> +        Update a given repository in repositories.Repositories() format
> +        """
> +        if len(repo) == 0:
> +            raise InvalidParameter("No parameters to update repository.")

MissingParameter()

> +
> +        self._repos.delete(repo_id)
> +        self.addRepo(repo)
> +
> +    def removeRepo(self, repo_id):
> +        """
> +        Remove a given repository
> +        """
> +        self._repos.delete(repo_id)
> +        self._removefromdisk(repo_id)
> +
> +    def _write2disk(self, repo={}):
> +        """
> +        Write repository info into disk.
> +        """
> +        # Get a list with all reposdir configured in system's YUM.
> +        conf_dir = self._conf.reposdir
> +        if not conf_dir:
> +            raise NotFoundError("There is no YUM configuration directory.")
> +
> +        # Generate the content to be wrote.
> +        repo_content = '[%s]\n' % repo['repo_id']
> +        repo_content = repo_content + 'name=%s\n' % repo['repo_name']
> +
> +        if not repo['baseurl']:
> +            if isinstance(repo['mirrors'], list):
> +                link = repo['mirrors'][0]
> +            else:
> +                link = repo['mirrors']
> +            repo_content = repo_content + 'mirrorlist=%s\n' % link
> +        else:
> +            if isinstance(repo['baseurl'], list):
> +                link = repo['baseurl'][0]
> +            else:
> +                link = repo['baseurl']
> +            repo_content = repo_content + 'baseurl=%s\n' % link
> +
> +        if repo['enabled']:
> +            repo_content = repo_content + 'enabled=1\n'
> +        else:
> +            repo_content = repo_content + 'enabled=0\n'
> +
> +        if repo['gpgcheck']:
> +            repo_content = repo_content + 'gpgcheck=1\n'
> +        else:
> +            repo_content = repo_content + 'gpgcheck=0\n'
> +
> +        if repo['gpgkey']:
> +            if isinstance(repo['gpgkey'], list):
> +                link = repo['gpgkey'][0]
> +            else:
> +                link = repo['gpgkey']
> +            repo_content = repo_content + 'gpgckey=%s\n' % link
> +
> +        # Scan for the confdirs and write the file in the first available
> +        # directory in the system. YUM will scan each confdir for repo files
> +        # and load it contents, so we can write in the first available dir.
> +        for dir in conf_dir:
> +            if os.path.isdir(dir):
> +                repo_file = dir + '/%s.repo' % repo['repo_id']
> +                if os.path.isfile(repo_file):
> +                    os.remove(repo_file)
> +
> +                try:
> +                    with open(repo_file, 'w') as fd:
> +                        fd.write(repo_content)
> +                        fd.close()
> +                except:
> +                    raise OperationFailed("Could not write repo file %s" %
> +                                          repo_file)
> +                break
> +        return True
> +
> +    def _removefromdisk(self, repo_id):
> +        """
> +        Delete the repo file from disk of a given repository
> +        """
> +        conf_dir = self._conf.reposdir
> +        if not conf_dir:
> +            raise NotFoundError("There is no YUM configuration directory.")
> +
> +        for dir in conf_dir:
> +            if os.path.isdir(dir):
> +                repo_file = dir + '/%s.repo' % repo_id
> +                if os.path.isfile(repo_file):
> +                    os.remove(repo_file)
> +
> +        return True




More information about the Kimchi-devel mailing list