May 2017 - Kimchi-devel - oVirt List Archives

[PATCH] [Kimchi] Remove federation feature from Kimchi as it is now on Wok
by Aline Manera 28 Jun '17

28 Jun '17

Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com> --- control/peers.py | 28 ---------- docs/API.md | 10 ---- docs/README-federation.md | 60 -------------------- kimchi.conf | 4 -- model/config.py | 8 +-- model/peers.py | 74 ------------------------ root.py | 8 --- tests/test_config.py.in | 3 +- tests/test_rest.py | 6 +- ui/js/kimchi.peers.js | 140 ---------------------------------------------- ui/js/src/kimchi.main.js | 3 - ui/pages/i18n.json.tmpl | 7 +-- 12 files changed, 7 insertions(+), 344 deletions(-) delete mode 100644 control/peers.py delete mode 100644 docs/README-federation.md delete mode 100644 model/peers.py delete mode 100644 ui/js/kimchi.peers.js diff --git a/control/peers.py b/control/peers.py deleted file mode 100644 index 5eaacdd..0000000 --- a/control/peers.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Project Kimchi -# -# Copyright IBM Corp, 2015-2017 -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -from wok.control.base import SimpleCollection -from wok.control.utils import UrlSubNode - - -@UrlSubNode("peers", True) -class Peers(SimpleCollection): - def __init__(self, model): - super(Peers, self).__init__(model) - self.admin_methods = ['GET'] diff --git a/docs/API.md b/docs/API.md index 3ecc7a0..1f67879 100644 --- a/docs/API.md +++ b/docs/API.md @@ -820,7 +820,6 @@ Contains information about Kimchi configuration. **Methods:** * **GET**: Retrieve configuration information - * federation: True if federation feature is enabled, False otherwise. * version: The version of the kimchi service * **POST**: *See Configuration Actions* @@ -1052,15 +1051,6 @@ List of available groups. * free: Amount of free space in the volume group. * size: Total size of the volume group. -### Collection: Peers - -**URI:** /plugins/kimchi/peers - -**Methods:** - -* **GET**: Return the list of Kimchi peers in the same network - (It uses openSLP for discovering) - ### Simple Collection: OVSBridges **URI:** /plugins/kimchi/ovsbridges diff --git a/docs/README-federation.md b/docs/README-federation.md deleted file mode 100644 index 3554757..0000000 --- a/docs/README-federation.md +++ /dev/null @@ -1,60 +0,0 @@ -Kimchi Project - Federation Feature -=================================== - -Federation feature is a Kimchi mechanism to discover Wok peers in the same -network. It uses openSLP tool (http://www.openslp.org/) to register and find Wok -servers. - -By default this feature is disabled on Wok as it is not critical for KVM -virtualization and requires additional software installation. - -To enable it, do the following: - -1. Install openslp and openslp-server rpm packages, - or install slpd and slptool deb packages. - -2. openSLP uses port 427 (UDP) and port 427 (TCP) so make sure to open those - ports in your firewall configuration - - For system using firewalld, do: - sudo firewall-cmd --permanent --add-port=427/udp - sudo firewall-cmd --permanent --add-port=427/tcp - sudo firewall-cmd --reload - - For openSUSE systems, do: - sudo /sbin/SuSEfirewall2 open EXT TCP 427 - sudo /sbin/SuSEfirewall2 open EXT UDP 427 - - For system using iptables, do: - sudo iptables -A INPUT -p tcp --dport 427 -j ACCEPT - sudo iptables -A INPUT -p udp --dport 427 -j ACCEPT - -3. In addition to the openSLP ports, you also need to allow multicast in the - firewall configuration - - For system using firewalld, do: - sudo firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -s <subnet> -j ACCEPT - - For openSUSE systems, do: - Add the subnet to the trusted networks listed on FW_TRUSTED_NETS in - /etc/sysconfig/SuSEfirewall2 file. - Make sure to restart /sbin/SuSEfirewall2 after modifying /etc/sysconfig/SuSEfirewall2 - - For system using iptables, do: - sudo iptables -A INPUT -s <subnet> -j ACCEPT - -4. Start slpd service and make sure it is up while running Wok - sudo service slpd start - -5. Enable federation on Wok by editing the /etc/wok/plugins.d/kimchi.conf file: - - federation = True - -6. Then start Wok service - sudo service wokd start - -The Wok server will be registered on openSLP on server starting up and will -be found by other Wok peers (with federation feature enabled) in the same -network. - -Enjoy! diff --git a/kimchi.conf b/kimchi.conf index 97a14bd..5e85e2b 100644 --- a/kimchi.conf +++ b/kimchi.conf @@ -3,9 +3,5 @@ enable = True [kimchi] -# Federation feature: register Wok server on openSLP and discover peers -# in the same network. Check README-federation for more details. -federation = False - # Automatically create ISO pool on server start up create_iso_pool = True diff --git a/model/config.py b/model/config.py index 78cdaeb..1db9afb 100644 --- a/model/config.py +++ b/model/config.py @@ -1,7 +1,7 @@ # # Project Kimchi # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -24,7 +24,7 @@ from wok.basemodel import Singleton from wok.exception import NotFoundError from wok.utils import run_command, wok_log -from wok.plugins.kimchi.config import config, find_qemu_binary +from wok.plugins.kimchi.config import find_qemu_binary from wok.plugins.kimchi.config import get_kimchi_version from wok.plugins.kimchi.distroloader import DistroLoader from wok.plugins.kimchi.model.featuretests import FeatureTests @@ -39,9 +39,7 @@ class ConfigModel(object): pass def lookup(self, name): - kconfig = config.get('kimchi', {}) - return {'federation': kconfig.get('federation', False), - 'version': get_kimchi_version()} + return {'version': get_kimchi_version()} class CapabilitiesModel(object): diff --git a/model/peers.py b/model/peers.py deleted file mode 100644 index 0a4ee4b..0000000 --- a/model/peers.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Project Kimchi -# -# Copyright IBM Corp, 2015-2017 -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -import cherrypy -import re -import socket - -from wok.config import config as wok_config -from wok.utils import run_command, wok_log - -from wok.plugins.kimchi.config import config - - -class PeersModel(object): - def __init__(self, **kargs): - # check federation feature is enabled on Kimchi server - if not config.get('kimchi', {}).get('federation', False): - return - - # register server on openslp - hostname = socket.getfqdn() - port = wok_config.get("server", "proxy_port") - self.url = hostname + ":" + port - - cmd = ["slptool", "register", - "service:wokd://%s" % self.url] - out, error, ret = run_command(cmd) - if out and len(out) != 0: - wok_log.error("Unable to register server on openSLP." - " Details: %s" % out) - cherrypy.engine.subscribe('exit', self._peer_deregister) - - def _peer_deregister(self): - cmd = ["slptool", "deregister", - "service:wokd://%s" % self.url] - out, error, ret = run_command(cmd) - if out and len(out) != 0: - wok_log.error("Unable to deregister server on openSLP." - " Details: %s" % out) - - def get_list(self): - # check federation feature is enabled on Kimchi server - if not config.get('kimchi', {}).get('federation', False): - return [] - - cmd = ["slptool", "findsrvs", "service:wokd"] - out, error, ret = run_command(cmd) - if ret != 0: - return [] - - peers = [] - for server in out.strip().split("\n"): - match = re.match("service:wokd://(.*?),.*", server) - peer = match.group(1) - if peer != self.url: - peers.append("https://" + peer) - - return peers diff --git a/root.py b/root.py index 4f131e7..d42e787 100644 --- a/root.py +++ b/root.py @@ -72,14 +72,6 @@ class Kimchi(WokRoot): self.domain = 'kimchi' self.messages = messages - self.extends = { - "/plugins/gingerbase": { - "host-dashboard.html": "/plugins/kimchi/js/kimchi.peers.js" - } - } - - self.depends = ['gingerbase'] - # Some paths or URI's present in the objectstore have changed after # Kimchi 2.0.0 release. Check here if an upgrade in the schema and data # are necessary. diff --git a/tests/test_config.py.in b/tests/test_config.py.in index 135097b..dd47130 100644 --- a/tests/test_config.py.in +++ b/tests/test_config.py.in @@ -80,8 +80,7 @@ class ConfigTests(unittest.TestCase): 'enable': True }, 'kimchi': { - 'federation': False, - 'create_iso_pool': True, + 'create_iso_pool': True }, '/': { 'tools.trailing_slash.on': False, diff --git a/tests/test_rest.py b/tests/test_rest.py index 852e4bd..f808478 100644 --- a/tests/test_rest.py +++ b/tests/test_rest.py @@ -1529,7 +1529,7 @@ class RestTests(unittest.TestCase): def test_config(self): resp = self.request('/plugins/kimchi/config').read() conf = json.loads(resp) - keys = ["federation", "version"] + keys = ["version"] self.assertEquals(keys, sorted(conf.keys())) def test_capabilities(self): @@ -1541,10 +1541,6 @@ class RestTests(unittest.TestCase): u'mem_hotplug_support', u'libvirtd_running'] self.assertEquals(sorted(keys), sorted(conf.keys())) - def test_peers(self): - resp = self.request('/plugins/kimchi/peers').read() - self.assertEquals([], json.loads(resp)) - def test_distros(self): resp = self.request('/plugins/kimchi/config/distros').read() distros = json.loads(resp) diff --git a/ui/js/kimchi.peers.js b/ui/js/kimchi.peers.js deleted file mode 100644 index 8b1a18d..0000000 --- a/ui/js/kimchi.peers.js +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Project Kimchi - * - * Copyright IBM Corp, 2016 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var kimchi = { - - getPeers: function(suc, err) { - wok.requestJSON({ - url: 'plugins/kimchi/peers', - type: 'GET', - contentType: 'application/json', - dataType: 'json', - resend: true, - success: suc, - error: err ? err : function(data) { - wok.message.error(data.responseJSON.reason); - } - }); - }, - - initPeers: function() { - - var peersAccordion = "<div class='panel-group federation-enabled accordion' id='peers-content-area-accordion' role='tablist' aria-multiselectable='true'>" + - "<h3>" + - "<a role='button' aria-expanded='true' data-toggle='collapse' data-parent='#peers-content-area-accordion' href='#peers-content-area' aria-expanded='false' aria-controls='peers-content-area' class=''>" + - "<span class='accordion-icon'></span>" + - "<span class='accordion-text' id='#peers-title'>"+i18n['KCHPEERS0001M']+"</span>" + - "</a>" + - "</h3>" + - "<div id='peers-content-area' class='panel-collapse collapse in' role='tabpanel' aria-labelledby='peers-title'>" + - "<div id='peers-alert-container'></div>" + - "<div class='row'>" + - "<div class='col-sm-12'>" + - "<table id='peers-list' class='table table-striped' cellspacing='0' width='100%''>" + - "<thead>" + - "<tr>" + - "<th><span class='sr-only'>" + i18n['KCHPEERS0001M'] + "</span></th>" + - "</tr>" + - "</thead>" + - "</table>" + - "</div>" + - "</div>" + - "<div class='wok-mask' role='presentation'>" + - "<div class='wok-mask-loader-container'>" + - "<div class='wok-mask-loading'>" + - "<div class='wok-mask-loading-icon'></div>" + - "<div class='wok-mask-loading-text'>" + i18n['WOKGRD6001M'] + "</div>" + - "</div>" + - "</div>" + - "</div>" + - "</div>" + - "</div>"; - - var peersDatatableTable; - var peers = new Array(); - - $('#peers-container > .container').append(peersAccordion); - - var peersDatatable = function(nwConfigDataSet) { - peersDatatableTable = $('#peers-list').DataTable({ - "processing": true, - "data": peers, - "language": { - "emptyTable": i18n['WOKSETT0010M'] - }, - "order": [], - "paging": false, - "dom": '<"row"<"col-sm-12"t>>', - "scrollY": "269px", - "scrollCollapse": true, - "columnDefs": [{ - "targets": 0, - "searchable": false, - "orderable": false, - "width": "100%", - "className": "tabular-data", - "render": function(data, type, full, meta) { - return '<a href="' + data + '" target="_blank">' + data + '</a>'; - } - }], - "initComplete": function(settings, json) { - $('#peers-content-area > .wok-mask').addClass('hidden'); - } - }); - }; - - var getPeers = function() { - kimchi.getPeers(function(result) { - peers.length = 0; - for (var i = 0; i < result.length; i++) { - var tempArr = []; - tempArr.push(result[i]); - peers.push(tempArr); - } - peersDatatable(peers); - }, function(err) { - wok.message.error(err.responseJSON.reason, '#peers-alert-container', true); - }); - }; - getPeers(); - - }, - - getConfig: function(suc, err, done) { - done = typeof done !== 'undefined' ? done : function() {}; - wok.requestJSON({ - url: "plugins/kimchi/config", - type: "GET", - contentType: "application/json", - dataType: "json", - success: suc, - error: err, - complete: done - }); - } -} - -$(document).ready(function() { - // Peers check - kimchi.getConfig(function(config) { - if (config.federation) { - $("#host-content-container").after('<div id="peers-container"><div class="container"></div></div>'); - kimchi.initPeers(); - } - }); -}); diff --git a/ui/js/src/kimchi.main.js b/ui/js/src/kimchi.main.js index 47531ae..89df799 100644 --- a/ui/js/src/kimchi.main.js +++ b/ui/js/src/kimchi.main.js @@ -19,9 +19,6 @@ kimchi.config = undefined; kimchi.getConfig(function(result) { kimchi.config = result; - - if(kimchi.config.federation == true) - $('#peers').removeClass('hide-content'); }, function() { kimchi.config = {} }); diff --git a/ui/pages/i18n.json.tmpl b/ui/pages/i18n.json.tmpl index 4510921..d44d4bd 100644 --- a/ui/pages/i18n.json.tmpl +++ b/ui/pages/i18n.json.tmpl @@ -1,7 +1,7 @@ #* * Project Kimchi * - * Copyright IBM Corp, 2014-2016 + * Copyright IBM Corp, 2014-2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -154,8 +154,5 @@ "KCHVMSTOR0007M": "$_("qed")", "KCHVMSTOR0008M": "$_("raw")", "KCHVMSTOR0009M": "$_("vmdk")", - "KCHVMSTOR0010M": "$_("vpc")", - - - "KCHPEERS0001M": "$_("Peers")" + "KCHVMSTOR0010M": "$_("vpc")" } -- 2.9.3

2 2

[PATCH] [Kimchi] Remove ginger-base as Kimchi dependency
by Aline Manera 28 Jun '17

28 Jun '17

Depends on: [PATCH] [Kimchi] Remove federation feature from Kimchi as it is now on Wok Aline Manera (1): Remove ginger-base as Kimchi dependency contrib/DEBIAN/control.in | 1 - contrib/kimchi.spec.fedora.in | 1 - contrib/kimchi.spec.suse.in | 1 - docs/README.md | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) -- 2.9.3

2 3

[PATCH] [WoK] Bug fix #217: Fixing select() waking up
by dhbarboza82＠gmail.com 18 May '17

18 May '17

From: Daniel Henrique Barboza <danielhb(a)linux.vnet.ibm.com> There was a condition in the close of the Websocket that wasn't being covered properly by the pushserver prior to this patch. The fix was to put an additional condition to close our remote socket reference when we receive no data from it, instead of relying just in the Except call to do it. While we're at it, a cleanup was made in the code to remove a condition of a 'CLOSE' message that was being sent in the first versions of the Pushserver but didn't make upstream. Signed-off-by: Daniel Henrique Barboza <danielhb(a)linux.vnet.ibm.com> --- src/wok/pushserver.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/wok/pushserver.py b/src/wok/pushserver.py index 6cad2cf..8b16f3d 100644 --- a/src/wok/pushserver.py +++ b/src/wok/pushserver.py @@ -108,31 +108,27 @@ class PushServer(object): while self.server_running: read_ready, _, _ = select.select(self.connections, [], [], 1) + for sock in read_ready: if not self.server_running: break if sock == self.server_socket: - new_socket, addr = self.server_socket.accept() self.connections.append(new_socket) else: try: data = sock.recv(4096) - except: - try: + if not data: self.connections.remove(sock) - except ValueError: - pass - - continue - if data and data == 'CLOSE': - sock.send('ACK') + sock.close() + except: try: self.connections.remove(sock) except ValueError: pass - sock.close() + finally: + sock.close() except Exception as e: raise RuntimeError('Exception ocurred in listen() of pushserver ' -- 2.9.3

2 1

[PATCH] [Wok] Add federation feature to Wok
by Aline Manera 18 May '17

18 May '17

It was primarily implemented on Kimchi project. But it makes more sense to be on Wok since the idea is to allow user add/select on which server Wok should manage. For example, the user could have a Wok installation on server A to manage server B. Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com> --- docs/API/config.md | 2 ++ docs/API/peers.md | 20 +++++++++++ docs/README-federation.md | 60 ++++++++++++++++++++++++++++++++ docs/wokd.8.in | 6 +++- src/wok.conf.in | 4 +++ src/wok/config.py.in | 1 + src/wok/control/peers.py | 30 ++++++++++++++++ src/wok/model/config.py | 1 + src/wok/model/peers.py | 74 +++++++++++++++++++++++++++++++++++++++ src/wokd.in | 5 +++ tests/test_api.py | 6 +++- tests/test_config_model.py | 2 +- tests/test_server_root.py | 6 ++-- ui/js/src/wok.api.js | 14 ++++++++ ui/js/wok.peers.js | 75 ++++++++++++++++++++++++++++++++++++++++ ui/pages/tabs/settings.html.tmpl | 34 +++++++++++++++++- 16 files changed, 333 insertions(+), 7 deletions(-) create mode 100644 docs/API/peers.md create mode 100644 docs/README-federation.md create mode 100644 src/wok/control/peers.py create mode 100644 src/wok/model/peers.py create mode 100644 ui/js/wok.peers.js diff --git a/docs/API/config.md b/docs/API/config.md index d1d1007..0e47f5f 100644 --- a/docs/API/config.md +++ b/docs/API/config.md @@ -12,6 +12,8 @@ Contains information about the application environment and configuration. * proxy_port: SSL port to list on * websockets_port: Port for websocket proxy to listen on * auth: Authentication method used to log in to Wok + * server_root: Relative path to Wok server. No value is specified by default, ie, Wok will run on '/' + * federation: 'on' if federation feature is enabled, 'off' otherwise. * version: Wok version * **POST**: *See Task Actions* diff --git a/docs/API/peers.md b/docs/API/peers.md new file mode 100644 index 0000000..fa5cf41 --- /dev/null +++ b/docs/API/peers.md @@ -0,0 +1,20 @@ +## REST API Specification for Peers + +### Collection: Peers + +**URI:** /peers + +Return a list of Wok peers in the same network. +(It uses openSLP for discovering) + +**Methods:** + +* **GET**: Retrieve a list peers URLs. + +#### Examples +GET /peers +[ + https://wok-peer0:8001, + https://wok-peer1:8001, + https://wok-peer2:8001, +] diff --git a/docs/README-federation.md b/docs/README-federation.md new file mode 100644 index 0000000..18e19c7 --- /dev/null +++ b/docs/README-federation.md @@ -0,0 +1,60 @@ +Wok Project - Federation Feature +=================================== + +Federation feature is a mechanism to discover Wok peers in the same +network. It uses openSLP tool (http://www.openslp.org/) to register and find Wok +servers. + +By default this feature is disabled on Wok as it is not critical for KVM +virtualization and requires additional software installation. + +To enable it, do the following: + +1. Install openslp and openslp-server rpm packages, + or install slpd and slptool deb packages. + +2. openSLP uses port 427 (UDP) and port 427 (TCP) so make sure to open those + ports in your firewall configuration + + For system using firewalld, do: + sudo firewall-cmd --permanent --add-port=427/udp + sudo firewall-cmd --permanent --add-port=427/tcp + sudo firewall-cmd --reload + + For openSUSE systems, do: + sudo /sbin/SuSEfirewall2 open EXT TCP 427 + sudo /sbin/SuSEfirewall2 open EXT UDP 427 + + For system using iptables, do: + sudo iptables -A INPUT -p tcp --dport 427 -j ACCEPT + sudo iptables -A INPUT -p udp --dport 427 -j ACCEPT + +3. In addition to the openSLP ports, you also need to allow multicast in the + firewall configuration + + For system using firewalld, do: + sudo firewall-cmd --direct --add-rule ipv4 filter INPUT 0 -s <subnet> -j ACCEPT + + For openSUSE systems, do: + Add the subnet to the trusted networks listed on FW_TRUSTED_NETS in + /etc/sysconfig/SuSEfirewall2 file. + Make sure to restart /sbin/SuSEfirewall2 after modifying /etc/sysconfig/SuSEfirewall2 + + For system using iptables, do: + sudo iptables -A INPUT -s <subnet> -j ACCEPT + +4. Start slpd service and make sure it is up while running Wok + sudo service slpd start + +5. Enable federation on Wok by editing the /etc/wok/wok.conf file: + + federation = on + +6. Then start Wok service + sudo service wokd start + +The Wok server will be registered on openSLP on server starting up and will +be found by other Wok peers (with federation feature enabled) in the same +network. + +Enjoy! diff --git a/docs/wokd.8.in b/docs/wokd.8.in index 217d9f6..09b5847 100644 --- a/docs/wokd.8.in +++ b/docs/wokd.8.in @@ -7,7 +7,7 @@ wokd - Launch Wok web server [\fB--cherrypy_port\fP \fICHERRYPY_PORT\fP] [\fB--websockets_port\fP \fIWEBSOCKETS_PORT\fP] [\fB--session_timeout\fP \fISESSION_TIMEOUT\fP] [\fB--log_level\fP \fILOG_LEVEL\fP] [\fB--log_dir\fP \fILOG_DIR\fP] [\fB--environment\fP \fIENV\fP] -[\fB--server_root\fP \fISERVER_ROOT\fP] [\fB--test\fP] +[\fB--server_root\fP \fISERVER_ROOT\fP] [\fB--federation\fP \fIfederation\fP] [\fB--test\fP] .SH DESCRIPTION \fBWok\fP is a cherrypy-based web framework with HTML5 support originated from Kimchi. It can be extended by plugins which expose functionality through REST APIs. @@ -55,6 +55,10 @@ Check cherrypy documentation for more details (default \fIproduction\fP). \fB\-\-server_root\fP [\fISERVER_ROOT\fP] Relative path to Wok server. No value is specified by default, ie, Wok will run on '/'. .TP +\fB\-\-federation\fP [\fIon\fP | \fIoff\fP] +Register and discover Wok peers in the same network using OpenSLP. Check +below the \fBREAD-federation\fP for more details (default \fIoff\fP). +.TP \fB\-\-test\fP Run Wok on a mock version that does not affect the system. For testing proposals. It depends on how plugins implements the mock environment as well. diff --git a/src/wok.conf.in b/src/wok.conf.in index 1ebdacf..1ed6310 100644 --- a/src/wok.conf.in +++ b/src/wok.conf.in @@ -27,6 +27,10 @@ # uncomment the following: #server_root=/wok +# Federation feature: register Wok server on openSLP and discover peers +# in the same network. Check README-federation for more details. +#federation = off + [logging] # Log directory diff --git a/src/wok/config.py.in b/src/wok/config.py.in index bbf2c09..39d7ad2 100644 --- a/src/wok/config.py.in +++ b/src/wok/config.py.in @@ -277,6 +277,7 @@ def _get_config(): config.set("server", "environment", "production") config.set('server', 'max_body_size', '4*1024*1024') config.set("server", "server_root", "") + config.set("server", "federation", "off") config.set("server", "test", "") config.add_section("authentication") config.set("authentication", "method", "pam") diff --git a/src/wok/control/peers.py b/src/wok/control/peers.py new file mode 100644 index 0000000..2a14815 --- /dev/null +++ b/src/wok/control/peers.py @@ -0,0 +1,30 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2017 +# +# Code derived from Kimchi Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from wok.control.base import SimpleCollection +from wok.control.utils import UrlSubNode + + +@UrlSubNode("peers", True) +class Peers(SimpleCollection): + def __init__(self, model): + super(Peers, self).__init__(model) + self.admin_methods = ['GET'] diff --git a/src/wok/model/config.py b/src/wok/model/config.py index b69f2dd..887ceba 100644 --- a/src/wok/model/config.py +++ b/src/wok/model/config.py @@ -34,6 +34,7 @@ class ConfigModel(object): 'websockets_port': config.get('server', 'websockets_port'), 'auth': config.get('authentication', 'method'), 'server_root': config.get('server', 'server_root'), + 'federation': config.get('server', 'federation'), 'version': get_version()} def reload(self, name): diff --git a/src/wok/model/peers.py b/src/wok/model/peers.py new file mode 100644 index 0000000..02afd59 --- /dev/null +++ b/src/wok/model/peers.py @@ -0,0 +1,74 @@ +# +# Project Wok +# +# Copyright IBM Corp, 2017 +# +# Code derived from Kimchi Project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import cherrypy +import re +import socket + +from wok.config import config +from wok.utils import run_command, wok_log + + +class PeersModel(object): + def __init__(self, **kargs): + # check federation feature is enabled on Wok server + if not config.get('server', 'federation') == 'off': + return + + # register server on openslp + hostname = socket.getfqdn() + port = config.get("server", "proxy_port") + self.url = hostname + ":" + port + + cmd = ["slptool", "register", + "service:wokd://%s" % self.url] + out, error, ret = run_command(cmd) + if out and len(out) != 0: + wok_log.error("Unable to register server on openSLP." + " Details: %s" % out) + cherrypy.engine.subscribe('exit', self._peer_deregister) + + def _peer_deregister(self): + cmd = ["slptool", "deregister", + "service:wokd://%s" % self.url] + out, error, ret = run_command(cmd) + if out and len(out) != 0: + wok_log.error("Unable to deregister server on openSLP." + " Details: %s" % out) + + def get_list(self): + # check federation feature is enabled on Wok server + if not config.get('server', 'federation') == 'off': + return [] + + cmd = ["slptool", "findsrvs", "service:wokd"] + out, error, ret = run_command(cmd) + if ret != 0: + return [] + + peers = [] + for server in out.strip().split("\n"): + match = re.match("service:wokd://(.*?),.*", server) + peer = match.group(1) + if peer != self.url: + peers.append("https://" + peer) + + return peers diff --git a/src/wokd.in b/src/wokd.in index 29586a0..dfe85a8 100644 --- a/src/wokd.in +++ b/src/wokd.in @@ -48,6 +48,7 @@ def main(options): websockets_port = config.config.get("server", "websockets_port") session_timeout = config.config.get("server", "session_timeout") runningEnv = config.config.get("server", "environment") + federation = config.config.get("server", "federation") server_root = config.config.get("server", "server_root") logDir = config.config.get("logging", "log_dir") logLevel = config.config.get("logging", "log_level") @@ -71,6 +72,10 @@ def main(options): help="Running environment of wok server") parser.add_option('--server_root', type="string", default=server_root, help="Relative path to server") + parser.add_option('--federation', default=federation, + help="Register and discover Wok peers in the same " + "network using openSLP. Check README-federation for" + " more details.") parser.add_option('--test', action='store_true', help="Run server in mock model") (options, args) = parser.parse_args() diff --git a/tests/test_api.py b/tests/test_api.py index 6fbee75..f978c93 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -49,11 +49,15 @@ class APITests(unittest.TestCase): def setUp(self): self.request = partial(utils.request) + def test_peers(self): + resp = self.request('/peers').read() + self.assertEquals([], json.loads(resp)) + def test_config(self): resp = self.request('/config').read() conf = json.loads(resp) keys = ["auth", "proxy_port", "websockets_port", "version", - "server_root"] + "server_root", "federation"] self.assertEquals(sorted(keys), sorted(conf.keys())) def test_config_plugins(self): diff --git a/tests/test_config_model.py b/tests/test_config_model.py index f8b0848..565b4df 100644 --- a/tests/test_config_model.py +++ b/tests/test_config_model.py @@ -30,7 +30,7 @@ class ConfigModelTests(unittest.TestCase): config = inst.config_lookup('') self.assertItemsEqual( ['proxy_port', 'websockets_port', 'auth', - 'server_root', 'version'], + 'server_root', 'version', 'federation'], config.keys() ) diff --git a/tests/test_server_root.py b/tests/test_server_root.py index e95a13b..89f34d3 100644 --- a/tests/test_server_root.py +++ b/tests/test_server_root.py @@ -1,7 +1,7 @@ # # Project Wok # -# Copyright IBM Corp, 2016 +# Copyright IBM Corp, 2016-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -49,7 +49,7 @@ class ServerRootTests(unittest.TestCase): # check if server_root in config is the same used to start server resp = request(server_root + '/config').read() conf = json.loads(resp) - self.assertEquals(len(conf), 5) + self.assertEquals(len(conf), 6) def test_development_env(self): """ @@ -61,4 +61,4 @@ class ServerRootTests(unittest.TestCase): # check if server_root in config is the same used to start server resp = request(server_root + '/config').read() conf = json.loads(resp) - self.assertEquals(len(conf), 5) + self.assertEquals(len(conf), 6) diff --git a/ui/js/src/wok.api.js b/ui/js/src/wok.api.js index 06b97aa..4e68eee 100644 --- a/ui/js/src/wok.api.js +++ b/ui/js/src/wok.api.js @@ -71,6 +71,20 @@ var wok = { }); }, + getPeers: function(suc, err) { + wok.requestJSON({ + url: 'peers', + type: 'GET', + contentType: 'application/json', + dataType: 'json', + resend: true, + success: suc, + error: err ? err : function(data) { + wok.message.error(data.responseJSON.reason); + } + }); + }, + getNotifications: function (suc, err) { wok.requestJSON({ url: 'notifications', diff --git a/ui/js/wok.peers.js b/ui/js/wok.peers.js new file mode 100644 index 0000000..67e58b0 --- /dev/null +++ b/ui/js/wok.peers.js @@ -0,0 +1,75 @@ +/* + * Project Wok + * + * Copyright IBM Corp, 2017 + * + * Code derived from Kimchi Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +wok.initPeers = function() { + var peersDatatableTable; + var peers = new Array(); + + var peersDatatable = function(nwConfigDataSet) { + peersDatatableTable = $('#peers-list').DataTable({ + "processing": true, + "data": peers, + "language": { + "emptyTable": i18n['WOKSETT0010M'] + }, + "order": [], + "paging": false, + "dom": '<"row"<"col-sm-12"t>>', + "scrollY": "269px", + "scrollCollapse": true, + "columnDefs": [{ + "targets": 0, + "searchable": false, + "orderable": false, + "width": "100%", + "className": "tabular-data", + "render": function(data, type, full, meta) { + return '<a href="' + data + '" target="_blank">' + data + '</a>'; + } + }], + "initComplete": function(settings, json) { + $('#peers-content-area > .wok-mask').addClass('hidden'); + } + }); + }; + + var getPeers = function() { + wok.getPeers(function(result) { + peers.length = 0; + for (var i = 0; i < result.length; i++) { + var tempArr = []; + tempArr.push(result[i]); + peers.push(tempArr); + } + peersDatatable(peers); + }, function(err) { + wok.message.error(err.responseJSON.reason, '#peers-alert-container', true); + }); + }; + getPeers(); + +} + +if (wok.config.federation == 'on') { + $("#peers-accordion").removeClass('hidden'); + wok.initPeers(); +} else { + $("#peers-accordion").addClass('hidden'); +} diff --git a/ui/pages/tabs/settings.html.tmpl b/ui/pages/tabs/settings.html.tmpl index ccc5b01..5f52839 100644 --- a/ui/pages/tabs/settings.html.tmpl +++ b/ui/pages/tabs/settings.html.tmpl @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA <head> <link rel="stylesheet" type="text/css" href="$href('css/settings.css')"> <script type="text/javascript" src="$href('js/wok.settings.js')"></script> + <script type="text/javascript" src="$href('js/wok.peers.js')"></script> <script type="text/javascript" src="$href('js/wok.bootgrid.js')"></script> </head> @@ -34,7 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA <div id="wok-root-container" class="wok"> <div class="container"> <div id="wokSettings" class="wok-settings"> -  +  <div class="panel-group accordion" id="plugins-mgmt-accordion" role="tablist" aria-multiselectable="false"> <h3> <a role="button" data-toggle="collapse" data-parent="#plugin-mgmt-accordion" href="#plugins-mgmt-content-area" aria-expanded="true" aria-controls="plugins-mgmt-content-area" class=""> @@ -65,6 +66,37 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA </div> </div>  +  + <div class='panel-group federation-enabled accordion hidden' id='peers-accordion' role='tablist' aria-multiselectable='true'> + <h3> + <a role='button' data-toggle='collapse' data-parent='#peers-accordion' href='#peers-content-area' aria-expanded='false' aria-controls='peers-content-area' class=''> + <span class='accordion-icon'></span><span class='accordion-text' id='#peers-title'>$_("Peers")</span> + </a> + </h3> + <div id='peers-content-area' class='panel-collapse collapse' role='tabpanel' aria-labelledby='peers-title'> + <div id='peers-alert-container'></div> + <div class='row'> + <div class='col-sm-12'> + <table id='peers-list' class='table table-striped' cellspacing='0' width='100%'> + <thead> + <tr> + <th><span class='sr-only'>$_("Peers")</span></th> + </tr> + </thead> + </table> + </div> + </div> + <div class='wok-mask' role='presentation'> + <div class='wok-mask-loader-container'> + <div class='wok-mask-loading'> + <div class='wok-mask-loading-icon'></div> + <div class='wok-mask-loading-text'>$_("Loading...")</div> + </div> + </div> + </div> + </div> + </div> +  </div> </div> </div> -- 2.9.3

2 2

[PATCH] [Kimchi] Move Kimchi specific functions from gingerbase.netinfo to Kimchi
by Aline Manera 09 May '17

09 May '17

The idea behind this patch is to eliminate the Ginger Base dependency. Almost all the functions in gingerbase.netinfo are for only Kimchi matters. So move them to Kimchi and when posible use ethtool module to provide the necessary info Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com> --- control/interfaces.py | 7 +- model/interfaces.py | 37 ++++- model/networks.py | 22 ++- model/ovsbridges.py | 4 +- network.py | 434 +++++++++++++++++++++++++++++++++++++++++++++++++- tests/test_model.py | 2 +- 6 files changed, 478 insertions(+), 28 deletions(-) diff --git a/control/interfaces.py b/control/interfaces.py index 7b6127a..7aba664 100644 --- a/control/interfaces.py +++ b/control/interfaces.py @@ -37,9 +37,4 @@ class Interface(Resource): @property def data(self): - return {'name': self.ident, - 'type': self.info['type'], - 'ipaddr': self.info['ipaddr'], - 'netmask': self.info['netmask'], - 'status': self.info['status'], - 'module': self.info['module']} + return self.info diff --git a/model/interfaces.py b/model/interfaces.py index 6be4356..0532072 100644 --- a/model/interfaces.py +++ b/model/interfaces.py @@ -1,7 +1,7 @@ # # Project Kimchi # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,11 +17,14 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import ethtool + from wok.exception import InvalidParameter, NotFoundError +from wok.stringutils import encode_value +from wok.utils import wok_log -from wok.plugins.gingerbase import netinfo +from wok.plugins.kimchi import network as netinfo from wok.plugins.kimchi.model.networks import NetworksModel -from wok.utils import wok_log class InterfacesModel(object): @@ -50,7 +53,29 @@ class InterfaceModel(object): pass def lookup(self, name): - try: - return netinfo.get_interface_info(name) - except ValueError: + if encode_value(name) not in map(encode_value, ethtool.get_devices()): raise NotFoundError("KCHIFACE0001E", {'name': name}) + + ipaddr = '' + netmask = '' + module = 'unknown' + status = 'down' + try: + ipaddr = ethtool.get_ipaddr(encode_value(name)) + netmask = ethtool.get_netmask(encode_value(name)) + module = ethtool.get_module(encode_value(name)) + + flags = ethtool.get_flags(encode_value(name)) + status = 'up' if flags & (ethtool.IFF_RUNNING | ethtool.IFF_UP) \ + else 'down' + except IOError: + pass + + iface_type = netinfo.get_interface_type(name) + + return {'name': name, + 'type': iface_type, + 'status': status, + 'ipaddr': ipaddr, + 'netmask': netmask, + 'module': module} diff --git a/model/networks.py b/model/networks.py index 59b111b..722b97b 100644 --- a/model/networks.py +++ b/model/networks.py @@ -28,10 +28,7 @@ from wok.exception import MissingParameter, NotFoundError, OperationFailed from wok.utils import run_command, wok_log from wok.xmlutils.utils import xpath_get_text -from wok.plugins.gingerbase import netinfo -from wok.plugins.gingerbase.netinfo import get_vlan_device, is_bridge, is_vlan -from wok.plugins.gingerbase.netinfo import ports -from wok.plugins.kimchi import network as knetwork +from wok.plugins.kimchi import network as netinfo from wok.plugins.kimchi.config import kimchiPaths from wok.plugins.kimchi.model.featuretests import FeatureTests from wok.plugins.kimchi.osinfo import defaults as tmpl_defaults @@ -130,8 +127,8 @@ class NetworksModel(object): xml = network.XMLDesc(0) subnet = NetworkModel.get_network_from_xml(xml)['subnet'] subnet and invalid_addrs.append(ipaddr.IPNetwork(subnet)) - addr_pools = addr_pools if addr_pools else knetwork.PrivateNets - return knetwork.get_one_free_network(invalid_addrs, addr_pools) + addr_pools = addr_pools if addr_pools else netinfo.PrivateNets + return netinfo.get_one_free_network(invalid_addrs, addr_pools) def _set_network_subnet(self, params): netaddr = params.get('subnet', '') @@ -281,7 +278,7 @@ class NetworksModel(object): conn = self.conn.get() if iface_xml is None: try: - mac = knetwork.get_dev_macaddr(str(interface)) + mac = netinfo.get_dev_macaddr(str(interface)) iface_xml = get_iface_xml({'type': 'ethernet', 'name': interface, 'mac': mac, @@ -364,7 +361,7 @@ class NetworkModel(object): connection = 'macvtap' # exposing the network on linux bridge or macvtap interface - interface_subnet = knetwork.get_dev_netaddr(interface) + interface_subnet = netinfo.get_dev_netaddr(interface) subnet = subnet if subnet else interface_subnet # libvirt use format 192.168.0.1/24, standard should be 192.168.0.0/24 @@ -536,10 +533,11 @@ class NetworkModel(object): # get target device if bridge was created by Kimchi if connection == 'bridge': iface = info['interfaces'][0] - if is_bridge(iface) and iface.startswith(KIMCHI_BRIDGE_PREFIX): - port = ports(iface)[0] - if is_vlan(port): - dev = get_vlan_device(port) + if (netinfo.is_bridge(iface) and + iface.startswith(KIMCHI_BRIDGE_PREFIX)): + port = netinfo.ports(iface)[0] + if netinfo.is_vlan(port): + dev = netinfo.get_vlan_device(port) info['interfaces'] = original['interfaces'] = [dev] # nic else: diff --git a/model/ovsbridges.py b/model/ovsbridges.py index 212520f..f5bb3df 100644 --- a/model/ovsbridges.py +++ b/model/ovsbridges.py @@ -1,7 +1,7 @@ # # Project Kimchi # -# Copyright IBM Corp, 2016 +# Copyright IBM Corp, 2016-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -17,7 +17,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from wok.plugins.gingerbase.netinfo import ovs_bridges +from wok.plugins.kimchi.network import ovs_bridges class OVSBridgesModel(object): diff --git a/network.py b/network.py index ec567ca..69944e7 100644 --- a/network.py +++ b/network.py @@ -1,7 +1,7 @@ # # Project Kimchi # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -19,7 +19,13 @@ # import ethtool +import glob import ipaddr +import os +from distutils.spawn import find_executable + +from wok.stringutils import encode_value +from wok.utils import run_command APrivateNets = ipaddr.IPNetwork("10.0.0.0/8") @@ -30,6 +36,432 @@ DefaultNetsPool = [ipaddr.IPNetwork('192.168.122.0/23'), ipaddr.IPNetwork('192.168.124.0/22'), ipaddr.IPNetwork('192.168.128.0/17')] +NET_PATH = '/sys/class/net' +NIC_PATH = '/sys/class/net/*/device' +BRIDGE_PATH = '/sys/class/net/*/bridge' +BONDING_PATH = '/sys/class/net/*/bonding' +WLAN_PATH = '/sys/class/net/*/wireless' +NET_BRPORT = '/sys/class/net/%s/brport' +NET_MASTER = '/sys/class/net/%s/master' +PROC_NET_VLAN = '/proc/net/vlan/' +BONDING_SLAVES = '/sys/class/net/%s/bonding/slaves' +BRIDGE_PORTS = '/sys/class/net/%s/brif' + + +def wlans(): + """Get all wlans declared in /sys/class/net/*/wireless. + + Returns: + List[str]: a list with the wlans found. + + """ + return [b.split('/')[-2] for b in glob.glob(WLAN_PATH)] + + +def nics(): + """Get all nics of the host. + + This function returns every nic, including those + that might be loaded from an usb port. + + Returns: + List[str]: a list with the nics found. + + """ + return list(set([b.split('/')[-2] for b in glob.glob(NIC_PATH)]) - + set(wlans())) + + +def is_nic(iface): + """Checks if iface is a nic. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a nic, False otherwise. + + """ + return encode_value(iface) in map(encode_value, nics()) + + +def bondings(): + """Get all bondings of the host. + + Returns: + List[str]: a list with the bonds found. + + """ + return [b.split('/')[-2] for b in glob.glob(BONDING_PATH)] + + +def is_bonding(iface): + """Checks if iface is a bond. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a bond, False otherwise. + + """ + return encode_value(iface) in map(encode_value, bondings()) + + +def vlans(): + """Get all vlans of the host. + + Returns: + List[str]: a list with the vlans found. + + """ + return list(set([b.split('/')[-1] + for b in glob.glob(NET_PATH + '/*')]) & + set([b.split('/')[-1] + for b in glob.glob(PROC_NET_VLAN + '*')])) + + +def is_vlan(iface): + """Checks if iface is a vlan. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a vlan, False otherwise. + + """ + return encode_value(iface) in map(encode_value, vlans()) + + +def bridges(): + """Get all bridges of the host. + + Returns: + List[str]: a list with the bridges found. + + """ + return list(set([b.split('/')[-2] for b in glob.glob(BRIDGE_PATH)] + + ovs_bridges())) + + +def is_bridge(iface): + """Checks if iface is a bridge. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a bridge, False otherwise. + + """ + return encode_value(iface) in map(encode_value, bridges()) + + +def is_openvswitch_running(): + """Checks if the openvswitch service is running in the host. + + Returns: + bool: True if openvswitch service is running, False otherwise. + + """ + cmd = ['systemctl', 'is-active', 'openvswitch', '--quiet'] + _, _, r_code = run_command(cmd, silent=True) + return r_code == 0 + + +def ovs_bridges(): + """Get the OVS Bridges of the host. + + In some distributions, like Fedora, the files bridge and brif are + not created under /sys/class/net/<ovsbridge> for OVS bridges. + These specific functions allows one to differentiate OVS bridges + from other types of bridges. + + Returns: + List[str]: a list with the OVS bridges found. + + """ + if not is_openvswitch_running(): + return [] + + ovs_cmd = find_executable("ovs-vsctl") + + # openvswitch not installed: there is no OVS bridge configured + if ovs_cmd is None: + return [] + + out, _, r_code = run_command([ovs_cmd, 'list-br'], silent=True) + if r_code != 0: + return [] + + return [x.strip() for x in out.rstrip('\n').split('\n') if x.strip()] + + +def is_ovs_bridge(iface): + """Checks if iface is an OVS bridge. + + In some distributions, like Fedora, the files bridge and brif are + not created under /sys/class/net/<ovsbridge> for OVS bridges. + These specific functions allows one to differentiate OVS bridges + from other types of bridges. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is an OVS bridge, False otherwise. + + """ + return iface in ovs_bridges() + + +def ovs_bridge_ports(ovsbr): + """Get the ports of a OVS bridge. + + In some distributions, like Fedora, the files bridge and brif are + not created under /sys/class/net/<ovsbridge> for OVS bridges. + These specific functions allows one to differentiate OVS bridges + from other types of bridges. + + Args: + ovsbr (str): name of the OVS bridge + + Returns: + List[str]: a list with the ports of this bridge. + + """ + if not is_openvswitch_running(): + return [] + + ovs_cmd = find_executable("ovs-vsctl") + + # openvswitch not installed: there is no OVS bridge configured + if ovs_cmd is None: + return [] + + out, _, r_code = run_command([ovs_cmd, 'list-ports', ovsbr], silent=True) + if r_code != 0: + return [] + + return [x.strip() for x in out.rstrip('\n').split('\n') if x.strip()] + + +def all_interfaces(): + """Returns all interfaces of the host. + + Returns: + List[str]: a list with all interfaces of the host. + + """ + return [d.rsplit("/", 1)[-1] for d in glob.glob(NET_PATH + '/*')] + + +def slaves(bonding): + """Get all slaves from a bonding. + + Args: + bonding (str): the name of the bond. + + Returns: + List[str]: a list with all slaves. + + """ + with open(BONDING_SLAVES % bonding) as bonding_file: + res = bonding_file.readline().split() + return res + + +def ports(bridge): + """Get all ports from a bridge. + + Args: + bridge (str): the name of the OVS bridge. + + Returns: + List[str]: a list with all ports. + + """ + if bridge in ovs_bridges(): + return ovs_bridge_ports(bridge) + + return os.listdir(BRIDGE_PORTS % bridge) + + +def is_brport(nic): + """Checks if nic is a port of a bridge. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a port of a bridge, False otherwise. + + """ + ovs_brports = [] + + for ovsbr in ovs_bridges(): + ovs_brports += ovs_bridge_ports(ovsbr) + + return os.path.exists(NET_BRPORT % nic) or nic in ovs_brports + + +def is_bondlave(nic): + """Checks if nic is a bond slave. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a bond slave, False otherwise. + + """ + return os.path.exists(NET_MASTER % nic) + + +def operstate(dev): + """Get the operstate status of a device. + + Args: + dev (str): name of the device. + + Returns: + str: "up" or "down" + + """ + flags = ethtool.get_flags(encode_value(dev)) + return 'up' if flags & (ethtool.IFF_RUNNING | ethtool.IFF_UP) else 'down' + + +def get_vlan_device(vlan): + """ Return the device of the given VLAN. + + Args: + vlan (str): the vlan name. + + Returns: + str: the device of the VLAN. + + """ + dev = None + + if os.path.exists(PROC_NET_VLAN + vlan): + with open(PROC_NET_VLAN + vlan) as vlan_file: + for line in vlan_file: + if "Device:" in line: + dummy, dev = line.split() + break + return dev + + +def get_bridge_port_device(bridge): + """Return the nics list that belongs to a port of 'bridge'. + + Args: + bridge (str): the bridge name. + + Returns: + List[str]: the nic list. + + """ + # br --- v --- bond --- nic1 + if encode_value(bridge) not in map(encode_value, bridges()): + raise ValueError('unknown bridge %s' % bridge) + nics_list = [] + for port in ports(bridge): + if encode_value(port) in map(encode_value, vlans()): + device = get_vlan_device(port) + if encode_value(device) in map(encode_value, bondings()): + nics_list.extend(slaves(device)) + else: + nics_list.append(device) + if encode_value(port) in map(encode_value, bondings()): + nics_list.extend(slaves(port)) + else: + nics_list.append(port) + return nics_list + + +def aggregated_bridges(): + """Get the list of aggregated bridges of the host. + + Returns: + List[str]: the aggregated bridges list. + + """ + return [bridge for bridge in bridges() if + (set(get_bridge_port_device(bridge)) & set(nics()))] + + +def bare_nics(): + """Get the list of bare nics of the host. + + A nic is called bare when it is not a port of a bridge + or a slave of bond. + + Returns: + List[str]: the list of bare nics of the host. + + """ + return [nic for nic in nics() if not (is_brport(nic) or is_bondlave(nic))] + + +def is_bare_nic(iface): + """Checks if iface is a bare nic. + + Args: + iface (str): name of the interface. + + Returns: + bool: True if iface is a bare nic, False otherwise. + + """ + return encode_value(iface) in map(encode_value, bare_nics()) + + +# The nic will not be exposed when it is a port of a bridge or +# a slave of bond. +# The bridge will not be exposed when all it's port are tap. +def all_favored_interfaces(): + """Get the list of all favored interfaces of the host. + + The nic will not be exposed when it is a port of a bridge or + a slave of bond. The bridge will not be exposed when all its + port are tap. + + Returns: + List[str]: the list of favored interfaces. + + """ + return aggregated_bridges() + bare_nics() + bondings() + + +def get_interface_type(iface): + """Get the interface type of iface. + + Types supported: nic, bonding, bridge, vlan. If the type + can't be verified, 'unknown' is returned. + + Args: + iface (str): the interface name. + + Returns: + str: the interface type. + + """ + try: + if is_nic(iface): + return "nic" + if is_bonding(iface): + return "bonding" + if is_bridge(iface): + return "bridge" + if is_vlan(iface): + return "vlan" + return 'unknown' + except IOError: + return 'unknown' + def get_dev_macaddr(dev): info = ethtool.get_interfaces_info(dev)[0] diff --git a/tests/test_model.py b/tests/test_model.py index 95c9e08..5398d08 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -48,7 +48,7 @@ from wok.rollbackcontext import RollbackContext from wok.utils import convert_data_size from wok.xmlutils.utils import xpath_get_text -from wok.plugins.gingerbase import netinfo +from wok.plugins.kimchi import network as netinfo from wok.plugins.kimchi import osinfo from wok.plugins.kimchi.config import kimchiPaths as paths from wok.plugins.kimchi.model import model -- 2.9.3

2 2

[PATCH] [Kimchi] Add server arch from /host API instead of relying on Ginger Base
by Aline Manera 09 May '17

09 May '17

That is one more change related to remove Ginger Base dependency from Kimchi. Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com> --- control/host.py | 6 +++++- ui/js/src/kimchi.api.js | 6 +++--- ui/js/src/kimchi.main.js | 6 +++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/control/host.py b/control/host.py index b1204fe..7872895 100644 --- a/control/host.py +++ b/control/host.py @@ -17,6 +17,8 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import platform + from wok.control.base import Collection from wok.control.base import Resource, SimpleCollection from wok.control.utils import UrlSubNode @@ -25,6 +27,8 @@ from wok.exception import NotFoundError from wok.plugins.kimchi.control.cpuinfo import CPUInfo from wok.plugins.kimchi.utils import is_s390x +ARCH = platform.machine() + @UrlSubNode('host', True) class Host(Resource): @@ -39,7 +43,7 @@ class Host(Resource): @property def data(self): - return {} + return {'arch': ARCH} class VolumeGroups(Collection): diff --git a/ui/js/src/kimchi.api.js b/ui/js/src/kimchi.api.js index 1cb89c0..618d68b 100644 --- a/ui/js/src/kimchi.api.js +++ b/ui/js/src/kimchi.api.js @@ -1,7 +1,7 @@ /* * Project Kimchi * - * Copyright IBM Corp, 2013-2016 + * Copyright IBM Corp, 2013-2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1314,9 +1314,9 @@ var kimchi = { * Get the host information. */ - kimchi.getHostDetails = function(suc, err) { + kimchi.serverConfig = function(suc, err) { wok.requestJSON({ - url: 'plugins/gingerbase/host', + url: 'plugins/kimchi/host', type: 'GET', resend: true, contentType: 'application/json', diff --git a/ui/js/src/kimchi.main.js b/ui/js/src/kimchi.main.js index f3078ec..47531ae 100644 --- a/ui/js/src/kimchi.main.js +++ b/ui/js/src/kimchi.main.js @@ -1,7 +1,7 @@ /* * Project Kimchi * - * Copyright IBM Corp, 2013-2016 + * Copyright IBM Corp, 2013-2017 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,8 +34,8 @@ kimchi.getCapabilities(function(result) { }); kimchi.hostarch = undefined; -kimchi.getHostDetails(function(result) { - kimchi.hostarch = result["architecture"]; +kimchi.serverConfig(function(result) { + kimchi.hostarch = result["arch"]; }); $(function(){ -- 2.9.3

2 2

[PATCH] [Kimchi] Move Kimchi specific functions from gingerbase.disks to Kimchi
by Aline Manera 09 May '17

09 May '17

The idea behind this patch is to eliminate the Ginger Base dependency. Some functions in gingerbase.disks are for only Kimchi matters. Others are shared with Ginger and can not be rewritten using PyParted, each means they will be place on Kimchi and Ginger Base. As this code is mature enough, further changes will have few impact and the gain to incorporate Ginger Base into Ginger and eliminate a Kimchi dependency is bigger. Signed-off-by: Aline Manera <alinefm(a)linux.vnet.ibm.com> --- disks.py | 325 ++++++++++++++++++++++++++++++++++++++++++++++++++++ i18n.py | 5 + model/host.py | 4 +- tests/test_disks.py | 53 +++++++++ 4 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 disks.py create mode 100644 tests/test_disks.py diff --git a/disks.py b/disks.py new file mode 100644 index 0000000..e86ca2f --- /dev/null +++ b/disks.py @@ -0,0 +1,325 @@ +# +# Project Kimchi +# +# Copyright IBM Corp, 2015-2017 +# +# Code derived from Ginger Base project +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os.path +import re +from parted import Device as PDevice +from parted import Disk as PDisk + +from wok.exception import NotFoundError, OperationFailed +from wok.stringutils import encode_value +from wok.utils import run_command, wok_log + + +def _get_dev_node_path(maj_min): + """ Returns device node path given the device number 'major:min' """ + + dm_name = "/sys/dev/block/%s/dm/name" % maj_min + if os.path.exists(dm_name): + with open(dm_name) as dm_f: + content = dm_f.read().rstrip('\n') + return "/dev/mapper/" + content + + uevent = "/sys/dev/block/%s/uevent" % maj_min + with open(uevent) as ueventf: + content = ueventf.read() + + data = dict(re.findall(r'(\S+)=(".*?"|\S+)', content.replace("\n", " "))) + + return "/dev/%s" % data["DEVNAME"] + + +def _get_lsblk_devs(keys, devs=None): + if devs is None: + devs = [] + out, err, returncode = run_command( + ["lsblk", "-Pbo"] + [','.join(keys)] + devs + ) + if returncode != 0: + if 'not a block device' in err: + raise NotFoundError("KCHDISK00002E") + else: + raise OperationFailed("KCHDISK00001E", {'err': err}) + + return _parse_lsblk_output(out, keys) + + +def _get_dev_major_min(name): + maj_min = None + + keys = ["NAME", "MAJ:MIN"] + try: + dev_list = _get_lsblk_devs(keys) + except: + raise + + for dev in dev_list: + if dev['name'].split()[0] == name: + maj_min = dev['maj:min'] + break + else: + raise NotFoundError("KCHDISK00003E", {'device': name}) + + return maj_min + + +def _is_dev_leaf(devNodePath, name=None, devs=None, devtype=None): + try: + if devs and devtype != 'mpath': + for dev in devs: + if encode_value(name) == dev['pkname']: + return False + return True + # By default, lsblk prints a device information followed by children + # device information + childrenCount = len( + _get_lsblk_devs(["NAME"], [devNodePath])) - 1 + except OperationFailed as e: + # lsblk is known to fail on multipath devices + # Assume these devices contain children + wok_log.error( + "Error getting device info for %s: %s", devNodePath, e) + return False + + return childrenCount == 0 + + +def _is_dev_extended_partition(devType, devNodePath): + if devType != 'part': + return False + + if devNodePath.startswith('/dev/mapper'): + try: + dev_maj_min = _get_dev_major_min(devNodePath.split("/")[-1]) + parent_sys_path = '/sys/dev/block/' + dev_maj_min + '/slaves' + parent_dm_name = os.listdir(parent_sys_path)[0] + parent_maj_min = open( + parent_sys_path + + '/' + + parent_dm_name + + '/dev').readline().rstrip() + diskPath = _get_dev_node_path(parent_maj_min) + except Exception as e: + wok_log.error( + "Error dealing with dev mapper device: " + devNodePath) + raise OperationFailed("KCHDISK00001E", {'err': e.message}) + else: + diskPath = devNodePath.rstrip('0123456789') + + device = PDevice(diskPath) + try: + extended_part = PDisk(device).getExtendedPartition() + except NotImplementedError as e: + wok_log.warning( + "Error getting extended partition info for dev %s type %s: %s", + devNodePath, devType, e.message) + # Treate disk with unsupported partiton table as if it does not + # contain extended partitions. + return False + if extended_part and extended_part.path == devNodePath: + return True + return False + + +def _parse_lsblk_output(output, keys): + # output is on format key="value", + # where key can be NAME, TYPE, FSTYPE, SIZE, MOUNTPOINT, etc + lines = output.rstrip("\n").split("\n") + r = [] + for line in lines: + d = {} + for key in keys: + expression = r"%s=\".*?\"" % key + match = re.search(expression, line) + field = match.group() + k, v = field.split('=', 1) + d[k.lower()] = v[1:-1] + r.append(d) + return r + + +def _is_available(name, devtype, fstype, mountpoint, majmin, devs=None): + devNodePath = _get_dev_node_path(majmin) + # Only list unmounted and unformated and leaf and (partition or disk) + # leaf means a partition, a disk has no partition, or a disk not held + # by any multipath device. Physical volume belongs to no volume group + # is also listed. Extended partitions should not be listed. + if fstype == 'LVM2_member': + has_VG = True + else: + has_VG = False + if (devtype in ['part', 'disk', 'mpath'] and + fstype in ['', 'LVM2_member'] and + mountpoint == "" and + not has_VG and + _is_dev_leaf(devNodePath, name, devs, devtype) and + not _is_dev_extended_partition(devtype, devNodePath)): + return True + return False + + +def get_partitions_names(check=False): + names = set() + keys = ["NAME", "TYPE", "FSTYPE", "MOUNTPOINT", "MAJ:MIN"] + # output is on format key="value", + # where key can be NAME, TYPE, FSTYPE, MOUNTPOINT + for dev in _get_lsblk_devs(keys): + # split()[0] to avoid the second part of the name, after the + # whiteline + name = dev['name'].split()[0] + if check and not _is_available(name, dev['type'], dev['fstype'], + dev['mountpoint'], dev['maj:min']): + continue + names.add(name) + + return list(names) + + +def get_partition_details(name): + majmin = _get_dev_major_min(name) + dev_path = _get_dev_node_path(majmin) + + keys = ["TYPE", "FSTYPE", "SIZE", "MOUNTPOINT", "MAJ:MIN", "PKNAME"] + try: + dev = _get_lsblk_devs(keys, [dev_path])[0] + except: + wok_log.error("Error getting partition info for %s", name) + return {} + + dev['available'] = _is_available(name, dev['type'], dev['fstype'], + dev['mountpoint'], majmin) + if dev['mountpoint']: + # Sometimes the mountpoint comes with [SWAP] or other + # info which is not an actual mount point. Filtering it + regexp = re.compile(r"\[.*\]") + if regexp.search(dev['mountpoint']) is not None: + dev['mountpoint'] = '' + dev['path'] = dev_path + dev['name'] = name + return dev + + +def vgs(): + """ + lists all volume groups in the system. All size units are in bytes. + + [{'vgname': 'vgtest', 'size': 999653638144L, 'free': 0}] + """ + cmd = ['vgs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'vg_name,vg_size,vg_free'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHDISK00004E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of VGs + vgs = map(lambda v: v.strip(), out.strip('\n').split('\n')) + + # create a dict based on data retrieved from vgs + return map(lambda l: {'vgname': l[0], + 'size': long(l[1]), + 'free': long(l[2])}, + [fields.split() for fields in vgs]) + + +def lvs(vgname=None): + """ + lists all logical volumes found in the system. It can be filtered by + the volume group. All size units are in bytes. + + [{'lvname': 'lva', 'path': '/dev/vgtest/lva', 'size': 12345L}, + {'lvname': 'lvb', 'path': '/dev/vgtest/lvb', 'size': 12345L}] + """ + cmd = ['lvs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'lv_name,lv_path,lv_size,vg_name'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHDISK00004E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of LVs filtered by vgname, if + # provided + lvs = filter(lambda f: vgname is None or vgname in f, + map(lambda v: v.strip(), out.strip('\n').split('\n'))) + + # create a dict based on data retrieved from lvs + return map(lambda l: {'lvname': l[0], + 'path': l[1], + 'size': long(l[2])}, + [fields.split() for fields in lvs]) + + +def pvs(vgname=None): + """ + lists all physical volumes in the system. It can be filtered by the + volume group. All size units are in bytes. + + [{'pvname': '/dev/sda3', + 'size': 469502001152L, + 'uuid': 'kkon5B-vnFI-eKHn-I5cG-Hj0C-uGx0-xqZrXI'}, + {'pvname': '/dev/sda2', + 'size': 21470642176L, + 'uuid': 'CyBzhK-cQFl-gWqr-fyWC-A50Y-LMxu-iHiJq4'}] + """ + cmd = ['pvs', + '--units', + 'b', + '--nosuffix', + '--noheading', + '--unbuffered', + '--options', + 'pv_name,pv_size,pv_uuid,vg_name'] + + out, err, rc = run_command(cmd) + if rc != 0: + raise OperationFailed("KCHDISK00004E", {'err': err}) + + if not out: + return [] + + # remove blank spaces and create a list of PVs filtered by vgname, if + # provided + pvs = filter(lambda f: vgname is None or vgname in f, + map(lambda v: v.strip(), out.strip('\n').split('\n'))) + + # create a dict based on data retrieved from pvs + return map(lambda l: {'pvname': l[0], + 'size': long(l[1]), + 'uuid': l[2]}, + [fields.split() for fields in pvs]) diff --git a/i18n.py b/i18n.py index 4460ce5..7dc7b05 100644 --- a/i18n.py +++ b/i18n.py @@ -29,6 +29,11 @@ messages = { "KCHPART0001E": _("Partition %(name)s does not exist in the host"), + "KCHDISK00001E": _("Error while accessing dev mapper device, %(err)s"), + "KCHDISK00002E": _("Block device not found."), + "KCHDISK00003E": _("Block device %(device)s not found."), + "KCHDISK00004E": _("Unable to retrieve LVM information. Details: %(err)s"), + "KCHDEVS0001E": _('Unknown "_cap" specified'), "KCHDEVS0002E": _('"_passthrough" should be "true" or "false"'), "KCHDEVS0003E": _('"_passthrough_affected_by" should be a device name string'), diff --git a/model/host.py b/model/host.py index abf1191..90834e3 100644 --- a/model/host.py +++ b/model/host.py @@ -1,7 +1,7 @@ # # Project Kimchi # -# Copyright IBM Corp, 2015-2016 +# Copyright IBM Corp, 2015-2017 # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -26,7 +26,7 @@ from wok.exception import InvalidParameter from wok.exception import NotFoundError from wok.xmlutils.utils import xpath_get_text -from wok.plugins.gingerbase import disks +from wok.plugins.kimchi import disks from wok.plugins.kimchi.model import hostdev from wok.plugins.kimchi.model.config import CapabilitiesModel from wok.plugins.kimchi.model.vms import VMModel, VMsModel diff --git a/tests/test_disks.py b/tests/test_disks.py new file mode 100644 index 0000000..6782f55 --- /dev/null +++ b/tests/test_disks.py @@ -0,0 +1,53 @@ +# +# Project Kimchi +# +# Copyright IBM Corp, 2017 +# +# Code derived from Ginger Base +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import mock +import unittest + +from wok.exception import NotFoundError, OperationFailed +from wok.plugins.kimchi.disks import _get_lsblk_devs + + +class DiskTests(unittest.TestCase): + + @mock.patch('wok.plugins.kimchi.disks.run_command') + def test_lsblk_returns_404_when_device_not_found(self, mock_run_command): + mock_run_command.return_value = ["", "not a block device", 32] + fake_dev = "/not/a/true/block/dev" + keys = ["MOUNTPOINT"] + + with self.assertRaises(NotFoundError): + _get_lsblk_devs(keys, [fake_dev]) + cmd = ['lsblk', '-Pbo', 'MOUNTPOINT', fake_dev] + mock_run_command.assert_called_once_with(cmd) + + @mock.patch('wok.plugins.kimchi.disks.run_command') + def test_lsblk_returns_500_when_unknown_error_occurs( + self, mock_run_command): + + mock_run_command.return_value = ["", "", 1] + valid_dev = "/valid/block/dev" + keys = ["MOUNTPOINT"] + + with self.assertRaises(OperationFailed): + _get_lsblk_devs(keys, [valid_dev]) + cmd = ['lsblk', '-Pbo', 'MOUNTPOINT', valid_dev] + mock_run_command.assert_called_once_with(cmd) -- 2.9.3

2 2