Re: [ovirt-users] Issue with OVN/OVS and mandatory ovirtmgmt network

29 Dec 2016

      ------=_Part_5778955_901085841.1483014842363
Content-Type: text/plain; charset=utf-8
Content-Transfer-Encoding: 7bit

Hi,

The tunnels are created to connect multiple OVN controllers.
If there is only one, there is no need for the tunnels, so none
will be created, this is the correct behavior.

Does the problem still occur after setting configuring the OVN-controller? 

Marcin

----- Original Message -----
...
From: "Sverker Abrahamsson" <sverker@abrahamsson.com>
To: "Marcin Mirecki" <mmirecki@redhat.com>
Cc: "Ovirt Users" <users@ovirt.org>
Sent: Thursday, December 29, 2016 11:44:32 AM
Subject: Re: [ovirt-users] Issue with OVN/OVS and mandatory ovirtmgmt network
Hi
The rpm packages you listed in the other mail are installed but I had
not run vdsm-tool ovn-config to create tunnel as the OVN controller is
on the same host.
[root@h2 ~]# rpm -q openvswitch-ovn-common
openvswitch-ovn-common-2.6.90-1.el7.centos.x86_64
[root@h2 ~]# rpm -q openvswitch-ovn-host
openvswitch-ovn-host-2.6.90-1.el7.centos.x86_64
[root@h2 ~]# rpm -q python-openvswitch
python-openvswitch-2.6.90-1.el7.centos.noarch
After removing my manually created br-int and run
vdsm-tool ovn-config 127.0.0.1 172.27.1.1
then I have the br-int but 'ip link show' does not show any link
'genev_sys_' nor does 'ovs-vsctl show' any port for ovn. I assume these
are when there is an actual tunnel?
[root@h2 ~]# ovs-vsctl show
ebb6aede-cbbc-4f4f-a88a-a9cd72b2bd23
     Bridge br-int
         fail_mode: secure
         Port br-int
             Interface br-int
                 type: internal
     Bridge ovirtbridge
         Port ovirtbridge
             Interface ovirtbridge
                 type: internal
     Bridge "ovsbridge0"
         Port "ovsbridge0"
             Interface "ovsbridge0"
                 type: internal
         Port "eth0"
             Interface "eth0"
     ovs_version: "2.6.90"
[root@h2 ~]# ip link show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode
DEFAULT qlen 1
     link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast
master ovs-system state UP mode DEFAULT qlen 1000
     link/ether 44:8a:5b:84:7d:b3 brd ff:ff:ff:ff:ff:ff
3: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode
DEFAULT qlen 1000
     link/ether 5a:14:cf:28:47:e2 brd ff:ff:ff:ff:ff:ff
4: ovsbridge0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue
state UNKNOWN mode DEFAULT qlen 1000
     link/ether 44:8a:5b:84:7d:b3 brd ff:ff:ff:ff:ff:ff
5: br-int: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode
DEFAULT qlen 1000
     link/ether 9e:b0:3a:9d:f2:4b brd ff:ff:ff:ff:ff:ff
6: ovirtbridge: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue
state UNKNOWN mode DEFAULT qlen 1000
     link/ether a6:f6:e5:a4:5b:45 brd ff:ff:ff:ff:ff:ff
7: dummy0: <BROADCAST,NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue master
ovirtmgmt state UNKNOWN mode DEFAULT qlen 1000
     link/ether 66:e0:1c:c3:a9:d8 brd ff:ff:ff:ff:ff:ff
8: ovirtmgmt: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue
state UP mode DEFAULT qlen 1000
     link/ether 66:e0:1c:c3:a9:d8 brd ff:ff:ff:ff:ff:ff
Firewall settings:
[root@h2 ~]# firewall-cmd --list-all-zones
work
   target: default
   icmp-block-inversion: no
   interfaces:
   sources:
   services: dhcpv6-client ssh
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
drop
   target: DROP
   icmp-block-inversion: no
   interfaces:
   sources:
   services:
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
internal
   target: default
   icmp-block-inversion: no
   interfaces:
   sources:
   services: dhcpv6-client mdns samba-client ssh
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
external
   target: default
   icmp-block-inversion: no
   interfaces:
   sources:
   services: ssh
   ports:
   protocols:
   masquerade: yes
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
trusted
   target: ACCEPT
   icmp-block-inversion: no
   interfaces:
   sources:
   services:
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
home
   target: default
   icmp-block-inversion: no
   interfaces:
   sources:
   services: dhcpv6-client mdns samba-client ssh
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
dmz
   target: default
   icmp-block-inversion: no
   interfaces:
   sources:
   services: ssh
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
public (active)
   target: default
   icmp-block-inversion: no
   interfaces: eth0 ovsbridge0
   sources:
   services: dhcpv6-client ssh
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
block
   target: %%REJECT%%
   icmp-block-inversion: no
   interfaces:
   sources:
   services:
   ports:
   protocols:
   masquerade: no
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
ovirt (active)
   target: default
   icmp-block-inversion: no
   interfaces: ovirtbridge ovirtmgmt
   sources:
   services: dhcp ovirt-fence-kdump-listener ovirt-http ovirt-https
ovirt-imageio-proxy ovirt-postgres ovirt-provider-ovn
ovirt-vmconsole-proxy ovirt-websocket-proxy ssh vdsm
   ports:
   protocols:
   masquerade: yes
   forward-ports:
   sourceports:
   icmp-blocks:
   rich rules:
         rule family="ipv4" port port="6641" protocol="tcp" accept
         rule family="ipv4" port port="6642" protocol="tcp" accept
The db dump is attached
/Sverker
Den 2016-12-29 kl. 09:50, skrev Marcin Mirecki:
...
Hi,
Can you please do: "sudo ovsdb-client dump"
on the host and send me the output?
Have you configured the ovn controller to connect to the
OVN north? You can do it using "vdsm-tool ovn-config" or
using the OVN tools directly.
Please check out: https://www.ovirt.org/blog/2016/11/ovirt-provider-ovn/
for details.
Also please note that the OVN provider is completely different
from the neutron-openvswitch plugin. Please don't mix the two.
Marcin
----- Original Message -----
...
From: "Marcin Mirecki" <mmirecki@redhat.com>
To: "Sverker Abrahamsson" <sverker@abrahamsson.com>
Cc: "Ovirt Users" <users@ovirt.org>
Sent: Thursday, December 29, 2016 9:27:19 AM
Subject: Re: [ovirt-users] Issue with OVN/OVS and mandatory ovirtmgmt
network
Hi,
br-int is the OVN integration bridge, it should have been created
when installing OVN. I assume you have the following packages installed
on the host:
     openvswitch-ovn-common
     openvswitch-ovn-host
     python-openvswitch
Please give me some time to look at the connectivity problem.
Marcin
----- Original Message -----
...
From: "Sverker Abrahamsson" <sverker@abrahamsson.com>
To: "Marcin Mirecki" <mmirecki@redhat.com>
Cc: "Ovirt Users" <users@ovirt.org>
Sent: Thursday, December 29, 2016 12:47:04 AM
Subject: Re: [ovirt-users] Issue with OVN/OVS and mandatory ovirtmgmt
network
From
/usr/libexec/vdsm/hooks/before_device_create/ovirt_provider_ovn_hook
(installed by ovirt-provider-ovn-driver rpm):
BRIDGE_NAME = 'br-int'
Den 2016-12-28 kl. 23:56, skrev Sverker Abrahamsson:
...
Googling on the message about br-int suggested adding that bridge to
ovs:
ovs-vsctl add-br br-int
Then the VM is able to boot, but it fails to get network connectivity.
Output in /var/log/messages:
Dec 28 23:31:35 h2 ovs-vsctl: ovs|00001|vsctl|INFO|Called as ovs-vsctl
--timeout=5 -- --if-exists del-port vnet0 -- add-port br-int vnet0 --
set Interface vnet0 "external-ids:attached-mac=\"00:1a:4a:16:01:51\""
-- set Interface vnet0
"external-ids:iface-id=\"e8853aac-8a75-41b0-8010-e630017dcdd8\"" --
set Interface vnet0
"external-ids:vm-id=\"b9440d60-ef5a-4e2b-83cf-081df7c09e6f\"" -- set
Interface vnet0 external-ids:iface-status=active
Dec 28 23:31:35 h2 kernel: device vnet0 entered promiscuous mode
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -D PREROUTING -i vnet0 -j
libvirt-J-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -D POSTROUTING -o vnet0 -j
libvirt-P-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -L libvirt-J-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -L libvirt-P-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F libvirt-J-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X libvirt-J-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F libvirt-P-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X libvirt-P-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F J-vnet0-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X J-vnet0-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F J-vnet0-arp-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X J-vnet0-arp-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -D libvirt-out -m physdev
--physdev-is-bridged --physdev-out vnet0 -g FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -D libvirt-out -m physdev --physdev-out
vnet0 -g FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -D libvirt-in -m physdev --physdev-in vnet0
-g FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -D libvirt-host-in -m physdev --physdev-in
vnet0 -g HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -F FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -X FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -F FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -X FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -F HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -X HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -E FP-vnet0 FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -E FJ-vnet0 FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/iptables -w2 -w -E HJ-vnet0 HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -D libvirt-out -m physdev
--physdev-is-bridged --physdev-out vnet0 -g FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -D libvirt-out -m physdev --physdev-out
vnet0 -g FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -D libvirt-in -m physdev --physdev-in
vnet0 -g FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -D libvirt-host-in -m physdev --physdev-in
vnet0 -g HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -F FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -X FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -F FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -X FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -F HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -X HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -E FP-vnet0 FO-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -E FJ-vnet0 FI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ip6tables -w2 -w -E HJ-vnet0 HI-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -D PREROUTING -i vnet0 -j
libvirt-I-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -D POSTROUTING -o vnet0 -j
libvirt-O-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -L libvirt-I-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -L libvirt-O-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F libvirt-I-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X libvirt-I-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F libvirt-O-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X libvirt-O-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -L libvirt-P-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -E libvirt-P-vnet0
libvirt-O-vnet0' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F I-vnet0-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X I-vnet0-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -F I-vnet0-arp-mac' failed:
Dec 28 23:31:35 h2 firewalld: WARNING: COMMAND_FAILED:
'/usr/sbin/ebtables --concurrent -t nat -X I-vnet0-arp-mac' failed:
[root@h2 etc]# ovs-vsctl show
ebb6aede-cbbc-4f4f-a88a-a9cd72b2bd23
     Bridge ovirtbridge
         Port "ovirtport0"
             Interface "ovirtport0"
                 type: internal
         Port ovirtbridge
             Interface ovirtbridge
                 type: internal
     Bridge "ovsbridge0"
         Port "ovsbridge0"
             Interface "ovsbridge0"
                 type: internal
         Port "eth0"
             Interface "eth0"
     Bridge br-int
         Port br-int
             Interface br-int
                 type: internal
         Port "vnet0"
             Interface "vnet0"
     ovs_version: "2.6.90"
Searching through the code it appears that br-int comes from
neutron-openvswitch plugin ??
[root@h2 share]# rpm -qf
/usr/share/otopi/plugins/ovirt-host-deploy/openstack/neutron_openvswitch.py
ovirt-host-deploy-1.6.0-0.0.master.20161215101008.gitb76ad50.el7.centos.noarch
/Sverker
Den 2016-12-28 kl. 23:24, skrev Sverker Abrahamsson:
...
In addition I had to add an alias to modprobe:
[root@h2 modprobe.d]# cat dummy.conf
alias dummy0 dummy
Den 2016-12-28 kl. 23:03, skrev Sverker Abrahamsson:
> Hi
> I first tried to set device name to dummy_0, but then ifup did not
> succeed in creating the device unless I first did 'ip link add
> dummy_0 type dummy' but then it would not suceed to establish the if
> on reboot.
>
> Setting fake_nics = dummy0 would not work neither, but this works:
>
> fake_nics = dummy*
>
> The engine is now able to find the if and assign bridge ovirtmgmt to
> it.
>
> However, I then run into the next issue when starting a VM:
>
> 2016-12-28 22:28:23,897 ERROR
> [org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector]
> (ForkJoinPool-1-worker-2) [] Correlation ID: null, Call Stack: null,
> Custom Event ID: -1, Message: VM CentOS7 is down with error. Exit
> message: Cannot get interface MTU on 'br-int': No such device.
>
> This VM has a nic on ovirtbridge, which comes from the OVN provider.
>
> /Sverker
>
> Den 2016-12-28 kl. 14:38, skrev Marcin Mirecki:
>> Sverker,
>>
>> Can you try adding a vnic named veth_* or dummy_*,
>> (or alternatively add the name of the vnic to
>> vdsm.config fake_nics), and setup the management
>> network using this vnic?
>> I suppose adding the vnic you use for connecting
>> to the engine to fake_nics should make it visible
>> to the engine, and you should be able to use it for
>> the setup.
>>
>> Marcin
>>
>>
>>
>> ----- Original Message -----
>>> From: "Marcin Mirecki" <mmirecki@redhat.com>
>>> To: "Sverker Abrahamsson" <sverker@abrahamsson.com>
>>> Cc: "Ovirt Users" <users@ovirt.org>
>>> Sent: Wednesday, December 28, 2016 12:06:26 PM
>>> Subject: Re: [ovirt-users] Issue with OVN/OVS and mandatory
>>> ovirtmgmt network
>>>
>>>> I have an internal OVS bridge called ovirtbridge which has a port
>>>> with
>>>> IP address, but in the host network settings that port is not
>>>> visible.
>>> I just verified and unfortunately the virtual ports are not
>>> visible in engine
>>> to assign a network to :(
>>> I'm afraid that the engine is not ready for such a scenario (even
>>> if it
>>> works).
>>> Please give me some time to look for a solution.
>>>
>>> ----- Original Message -----
>>>> From: "Sverker Abrahamsson" <sverker@abrahamsson.com>
>>>> To: "Marcin Mirecki" <mmirecki@redhat.com>
>>>> Cc: "Ovirt Users" <users@ovirt.org>
>>>> Sent: Wednesday, December 28, 2016 11:48:24 AM
>>>> Subject: Re: [ovirt-users] Issue with OVN/OVS and mandatory
>>>> ovirtmgmt
>>>> network
>>>>
>>>> Hi Marcin
>>>> Yes, that is my issue. I don't want to let ovirt/vdsm see eth0 nor
>>>> ovsbridge0 since as soon as it sees them it messes up the network
>>>> config
>>>> so that the host will be unreachable.
>>>>
>>>> I have an internal OVS bridge called ovirtbridge which has a port
>>>> with
>>>> IP address, but in the host network settings that port is not
>>>> visible.
>>>> It doesn't help to name it ovirtmgmt.
>>>>
>>>> The engine is able to communicate with the host on the ip it has
>>>> been
>>>> given, it's just that it believes that it HAS to have a ovirtmgmt
>>>> network which can't be on OVN.
>>>>
>>>> /Sverker
>>>>
>>>>
>>>> Den 2016-12-28 kl. 10:45, skrev Marcin Mirecki:
>>>>> Hi Sverker,
>>>>>
>>>>> The management network is mandatory on each host. It's used by the
>>>>> engine to communicate with the host.
>>>>> Looking at your description and the exception it looks like it is
>>>>> missing.
>>>>> The error is caused by not having any network for the host
>>>>> (network list retrieved in
>>>>> InterfaceDaoImpl.getHostNetworksByCluster -
>>>>> which
>>>>> gets all the networks on nics for a host from vds_interface
>>>>> table in the
>>>>> DB).
>>>>>
>>>>> Could you maybe create a virtual nic connected to ovsbridge0 (as I
>>>>> understand you
>>>>> have no physical nic available) and use this for the management
>>>>> network?
>>>>>
>>>>>> I then create a bridge for use with ovirt, with a private
>>>>>> address.
>>>>> I'm not quite sure I understand. Is this yet another bridge
>>>>> connected to
>>>>> ovsbridge0?
>>>>> You could also attach the vnic for the management network here
>>>>> if need
>>>>> be.
>>>>>
>>>>> Please keep in mind that OVN has no use in setting up the
>>>>> management
>>>>> network.
>>>>> The OVN provider can only handle external networks, which can
>>>>> not be used
>>>>> for a
>>>>> management network.
>>>>>
>>>>> Marcin
>>>>>
>>>>>
>>>>> ----- Original Message -----
>>>>>> From: "Sverker Abrahamsson" <sverker@abrahamsson.com>
>>>>>> To: users@ovirt.org
>>>>>> Sent: Wednesday, December 28, 2016 12:39:59 AM
>>>>>> Subject: [ovirt-users] Issue with OVN/OVS and mandatory ovirtmgmt
>>>>>> network
>>>>>>
>>>>>>
>>>>>>
>>>>>> Hi
>>>>>> For long time I've been looking for proper support in ovirt for
>>>>>> Open
>>>>>> vSwitch
>>>>>> so I'm happy that it is moving in the right direction. However,
>>>>>> there
>>>>>> seems
>>>>>> to still be a dependency on a ovirtmgmt bridge and I'm unable
>>>>>> to move
>>>>>> that
>>>>>> to the OVN provider.
>>>>>>
>>>>>> The hosting center where I rent hw instances has a bit special
>>>>>> network
>>>>>> setup,
>>>>>> so I have one physical network port with a /32 netmask and
>>>>>> point-to-point
>>>>>> config to router. The physical port I connect to a ovs bridge
>>>>>> which has
>>>>>> the
>>>>>> public ip. Since ovirt always messes up the network config when
>>>>>> I've
>>>>>> tried
>>>>>> to let it have access to the network config for the physical
>>>>>> port, I've
>>>>>> set
>>>>>> eht0 and ovsbridge0 as hidden in vdsm.conf.
>>>>>>
>>>>>>
>>>>>> I then create a bridge for use with ovirt, with a private
>>>>>> address. With
>>>>>> the
>>>>>> OVN provider I am now able to import these into the engine and
>>>>>> it looks
>>>>>> good. When creating a VM I can select that it will have a vNic
>>>>>> on my OVS
>>>>>> bridge.
>>>>>>
>>>>>> However, I can't start the VM as an exception is thrown in the
>>>>>> log:
>>>>>>
>>>>>> 2016-12-28 00:13:33,350 ERROR
>>>>>> [org.ovirt.engine.core.bll.RunVmCommand]
>>>>>> (default task-5) [3c882d53] Error during ValidateFailure.:
>>>>>> java.lang.NullPointerException
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.scheduling.policyunits.NetworkPolicyUnit.validateRequiredNetworksAvailable(NetworkPolicyUnit.java:140)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.scheduling.policyunits.NetworkPolicyUnit.filter(NetworkPolicyUnit.java:69)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.scheduling.SchedulingManager.runInternalFilters(SchedulingManager.java:597)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.scheduling.SchedulingManager.runFilters(SchedulingManager.java:564)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.scheduling.SchedulingManager.canSchedule(SchedulingManager.java:494)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.validator.RunVmValidator.canRunVm(RunVmValidator.java:133)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.RunVmCommand.validate(RunVmCommand.java:940)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.CommandBase.internalValidate(CommandBase.java:886)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.CommandBase.validateOnly(CommandBase.java:366)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.PrevalidatingMultipleActionsRunner.canRunActions(PrevalidatingMultipleActionsRunner.java:113)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.PrevalidatingMultipleActionsRunner.invokeCommands(PrevalidatingMultipleActionsRunner.java:99)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.PrevalidatingMultipleActionsRunner.execute(PrevalidatingMultipleActionsRunner.java:76)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.Backend.runMultipleActionsImpl(Backend.java:613)
>>>>>>
>>>>>> [bll.jar:]
>>>>>> at
>>>>>> org.ovirt.engine.core.bll.Backend.runMultipleActions(Backend.java:583)
>>>>>>
>>>>>> [bll.jar:]
>>>>>>
>>>>>>
>>>>>> Looking at that section of code where the exception is thrown,
>>>>>> I see
>>>>>> that
>>>>>> it
>>>>>> iterates over host networks to find required networks, which I
>>>>>> assume is
>>>>>> ovirtmgmt. In the host network setup dialog I don't see any
>>>>>> networks at
>>>>>> all
>>>>>> but it lists ovirtmgmt as required. It also list the OVN
>>>>>> networks but
>>>>>> these
>>>>>> can't be statically assigned as they are added dynamically when
>>>>>> needed,
>>>>>> which is fine.
>>>>>>
>>>>>> I believe that I either need to remove ovirtmgmt network or
>>>>>> configure
>>>>>> that
>>>>>> it
>>>>>> is provided by the OVN provider, but neither is possible.
>>>>>> Preferably it
>>>>>> shouldn't be hardcoded which network is management and
>>>>>> mandatory but be
>>>>>> possible to configure.
>>>>>>
>>>>>> /Sverker
>>>>>> Den 2016-12-27 kl. 17:10, skrev Marcin Mirecki:
>>>>>>
>>>>>>
>>> _______________________________________________
>>> Users mailing list
>>> Users@ovirt.org
>>> http://lists.ovirt.org/mailman/listinfo/users
>>>
> _______________________________________________
> Users mailing list
> Users@ovirt.org
> http://lists.ovirt.org/mailman/listinfo/users
_______________________________________________
Users mailing list
Users@ovirt.org
http://lists.ovirt.org/mailman/listinfo/users

Users mailing list
Users@ovirt.org
http://lists.ovirt.org/mailman/listinfo/users
_______________________________________________
Users mailing list
Users@ovirt.org
http://lists.ovirt.org/mailman/listinfo/users
------=_Part_5778955_901085841.1483014842363
Content-Type: text/plain; charset=UTF-8; name=ovsdbdump
Content-Disposition: attachment; filename=ovsdbdump
Content-Transfer-Encoding: base64

QXV0b0F0dGFjaCB0YWJsZQpfdXVpZCBtYXBwaW5ncyBzeXN0ZW1fZGVzY3JpcHRpb24gc3lzdGVt
X25hbWUKLS0tLS0gLS0tLS0tLS0gLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tCgpCcmlk
Z2UgdGFibGUKX3V1aWQgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF1dG9fYXR0YWNo
IGNvbnRyb2xsZXIgZGF0YXBhdGhfaWQgICAgICAgIGRhdGFwYXRoX3R5cGUgZGF0YXBhdGhfdmVy
c2lvbiBleHRlcm5hbF9pZHMgZmFpbF9tb2RlIGZsb29kX3ZsYW5zIGZsb3dfdGFibGVzIGlwZml4
IG1jYXN0X3Nub29waW5nX2VuYWJsZSBtaXJyb3JzIG5hbWUgICAgICAgICBuZXRmbG93IG90aGVy
X2NvbmZpZyAgICAgICAgICAgICBwb3J0cyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb3RvY29scyByc3RwX2Vu
YWJsZSByc3RwX3N0YXR1cyBzZmxvdyBzdGF0dXMgc3RwX2VuYWJsZQotLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0gLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0t
LS0tLS0gLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLSAtLS0tLS0t
LS0gLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0gLS0tLS0gLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0t
LS0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0gLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0gLS0tLS0tLS0tIC0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tIC0tLS0tIC0t
LS0tLSAtLS0tLS0tLS0tCmQ0YWY1YmJiLWRmMTQtNGZmYy1hYWU5LWI2NTY2YTVmZmQ4NyBbXSAg
ICAgICAgICBbXSAgICAgICAgICIwMDAwNDQ4YTViODQ3ZGIzIiAiIiAgICAgICAgICAgICI8dW5r
bm93bj4iICAgICAge30gICAgICAgICAgIFtdICAgICAgICBbXSAgICAgICAgICB7fSAgICAgICAg
ICBbXSAgICBmYWxzZSAgICAgICAgICAgICAgICAgW10gICAgICAib3ZzYnJpZGdlMCIgW10gICAg
ICB7fSAgICAgICAgICAgICAgICAgICAgICAgW2E3Y2NhOGY1LTM0MzctNDNkYy04MzEwLTE5NTQ1
NGZiNzc3MSwgYjczNjFjNTctNDFhYS00YThmLWI2ZmUtNjdlNjQzMTI5YWNhXSBbXSAgICAgICAg
ZmFsc2UgICAgICAge30gICAgICAgICAgW10gICAge30gICAgIGZhbHNlICAgICAKOWQzYWIwOWUt
YTE0Ni00YmYyLWE1YmItYmE5OTQ4YzRmMmRkIFtdICAgICAgICAgIFtdICAgICAgICAgIjAwMDA5
ZWIwM2E5ZGYyNGIiICIiICAgICAgICAgICAgIjx1bmtub3duPiIgICAgICB7fSAgICAgICAgICAg
c2VjdXJlICAgIFtdICAgICAgICAgIHt9ICAgICAgICAgIFtdICAgIGZhbHNlICAgICAgICAgICAg
ICAgICBbXSAgICAgIGJyLWludCAgICAgICBbXSAgICAgIHtkaXNhYmxlLWluLWJhbmQ9InRydWUi
fSBbNDA2Y2FmNzItYTZmOS00ZmQ4LTgzZGMtMmJjNGZiMjE5NDRjXSAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgIFtdICAgICAgICBmYWxzZSAgICAgICB7fSAgICAgICAgICBb
XSAgICB7fSAgICAgZmFsc2UgICAgIAphNGU1ZjZhNS00ZWMxLTQ1NWItOThlOS04YjVlNmE4YjRj
YzQgW10gICAgICAgICAgW10gICAgICAgICAiMDAwMGE2ZjZlNWE0NWI0NSIgIiIgICAgICAgICAg
ICAiPHVua25vd24+IiAgICAgIHt9ICAgICAgICAgICBbXSAgICAgICAgW10gICAgICAgICAge30g
ICAgICAgICAgW10gICAgZmFsc2UgICAgICAgICAgICAgICAgIFtdICAgICAgb3ZpcnRicmlkZ2Ug
IFtdICAgICAge30gICAgICAgICAgICAgICAgICAgICAgIFs3YjQ1OTE3Yi1hYmFmLTRlYmQtYjUw
MS1jZTc2ZjA3ZmU2NWVdICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgW10g
ICAgICAgIGZhbHNlICAgICAgIHt9ICAgICAgICAgIFtdICAgIHt9ICAgICBmYWxzZSAgICAgCgpD
b250cm9sbGVyIHRhYmxlCl91dWlkIGNvbm5lY3Rpb25fbW9kZSBjb250cm9sbGVyX2J1cnN0X2xp
bWl0IGNvbnRyb2xsZXJfcmF0ZV9saW1pdCBlbmFibGVfYXN5bmNfbWVzc2FnZXMgZXh0ZXJuYWxf
aWRzIGluYWN0aXZpdHlfcHJvYmUgaXNfY29ubmVjdGVkIGxvY2FsX2dhdGV3YXkgbG9jYWxfaXAg
bG9jYWxfbmV0bWFzayBtYXhfYmFja29mZiBvdGhlcl9jb25maWcgcm9sZSBzdGF0dXMgdGFyZ2V0
Ci0tLS0tIC0tLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0t
LS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0t
LS0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0gLS0tLS0tLS0gLS0tLS0tLS0tLS0tLSAt
LS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0gLS0tLSAtLS0tLS0gLS0tLS0tCgpGbG93X1NhbXBsZV9D
b2xsZWN0b3JfU2V0IHRhYmxlCl91dWlkIGJyaWRnZSBleHRlcm5hbF9pZHMgaWQgaXBmaXgKLS0t
LS0gLS0tLS0tIC0tLS0tLS0tLS0tLSAtLSAtLS0tLQoKRmxvd19UYWJsZSB0YWJsZQpfdXVpZCBl
eHRlcm5hbF9pZHMgZmxvd19saW1pdCBncm91cHMgbmFtZSBvdmVyZmxvd19wb2xpY3kgcHJlZml4
ZXMKLS0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0gLS0tLS0tIC0tLS0gLS0tLS0tLS0tLS0t
LS0tIC0tLS0tLS0tCgpJUEZJWCB0YWJsZQpfdXVpZCBjYWNoZV9hY3RpdmVfdGltZW91dCBjYWNo
ZV9tYXhfZmxvd3MgZXh0ZXJuYWxfaWRzIG9ic19kb21haW5faWQgb2JzX3BvaW50X2lkIG90aGVy
X2NvbmZpZyBzYW1wbGluZyB0YXJnZXRzCi0tLS0tIC0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0t
LS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0gLS0tLS0t
LS0tLS0tIC0tLS0tLS0tIC0tLS0tLS0KCkludGVyZmFjZSB0YWJsZQpfdXVpZCAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgYWRtaW5fc3RhdGUgYmZkIGJmZF9zdGF0dXMgY2ZtX2ZhdWx0
IGNmbV9mYXVsdF9zdGF0dXMgY2ZtX2ZsYXBfY291bnQgY2ZtX2hlYWx0aCBjZm1fbXBpZCBjZm1f
cmVtb3RlX21waWRzIGNmbV9yZW1vdGVfb3BzdGF0ZSBkdXBsZXggZXJyb3IgZXh0ZXJuYWxfaWRz
IGlmaW5kZXggaW5ncmVzc19wb2xpY2luZ19idXJzdCBpbmdyZXNzX3BvbGljaW5nX3JhdGUgbGFj
cF9jdXJyZW50IGxpbmtfcmVzZXRzIGxpbmtfc3BlZWQgbGlua19zdGF0ZSBsbGRwIG1hYyBtYWNf
aW5fdXNlICAgICAgICAgIG10dSAgbXR1X3JlcXVlc3QgbmFtZSAgICAgICAgIG9mcG9ydCBvZnBv
cnRfcmVxdWVzdCBvcHRpb25zIG90aGVyX2NvbmZpZyBzdGF0aXN0aWNzICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhdHVzICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
dHlwZSAgICAKLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0t
IC0tLSAtLS0tLS0tLS0tIC0tLS0tLS0tLSAtLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0t
IC0tLS0tLS0tLS0gLS0tLS0tLS0gLS0tLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0tLS0tLS0g
LS0tLS0tIC0tLS0tIC0tLS0tLS0tLS0tLSAtLS0tLS0tIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0g
LS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLSAtLS0tLS0tLS0t
IC0tLS0tLS0tLS0gLS0tLSAtLS0gLS0tLS0tLS0tLS0tLS0tLS0tLSAtLS0tIC0tLS0tLS0tLS0t
IC0tLS0tLS0tLS0tLSAtLS0tLS0gLS0tLS0tLS0tLS0tLS0gLS0tLS0tLSAtLS0tLS0tLS0tLS0g
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tCmRiNGZmYWMwLWZkOTgtNDE0My05MWMw
LTNjYTQ3NjdlYmM1MiBkb3duICAgICAgICB7fSAge30gICAgICAgICBbXSAgICAgICAgW10gICAg
ICAgICAgICAgICBbXSAgICAgICAgICAgICBbXSAgICAgICAgIFtdICAgICAgIFtdICAgICAgICAg
ICAgICAgW10gICAgICAgICAgICAgICAgIFtdICAgICBbXSAgICB7fSAgICAgICAgICAgNSAgICAg
ICAwICAgICAgICAgICAgICAgICAgICAgIDAgICAgICAgICAgICAgICAgICAgICBbXSAgICAgICAg
ICAgMCAgICAgICAgICAgW10gICAgICAgICBkb3duICAgICAgIHt9ICAgW10gICI5ZTpiMDozYTo5
ZDpmMjo0YiIgMTUwMCBbXSAgICAgICAgICBici1pbnQgICAgICAgNjU1MzQgIFtdICAgICAgICAg
ICAgIHt9ICAgICAge30gICAgICAgICAgIHtjb2xsaXNpb25zPTAsIHJ4X2J5dGVzPTAsIHJ4X2Ny
Y19lcnI9MCwgcnhfZHJvcHBlZD0wLCByeF9lcnJvcnM9MCwgcnhfZnJhbWVfZXJyPTAsIHJ4X292
ZXJfZXJyPTAsIHJ4X3BhY2tldHM9MCwgdHhfYnl0ZXM9MCwgdHhfZHJvcHBlZD0wLCB0eF9lcnJv
cnM9MCwgdHhfcGFja2V0cz0wfSAgICAgICAgICAgICAgICB7ZHJpdmVyX25hbWU9b3BlbnZzd2l0
Y2h9ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpbnRlcm5h
bAoyMWRmN2E1OC1lNTcyLTQwNDItOGI2NC02MzcwZDhlNThmOTIgdXAgICAgICAgICAge30gIHt9
ICAgICAgICAgW10gICAgICAgIFtdICAgICAgICAgICAgICAgW10gICAgICAgICAgICAgW10gICAg
ICAgICBbXSAgICAgICBbXSAgICAgICAgICAgICAgIFtdICAgICAgICAgICAgICAgICBbXSAgICAg
W10gICAge30gICAgICAgICAgIDQgICAgICAgMCAgICAgICAgICAgICAgICAgICAgICAwICAgICAg
ICAgICAgICAgICAgICAgW10gICAgICAgICAgIDEgICAgICAgICAgIFtdICAgICAgICAgdXAgICAg
ICAgICB7fSAgIFtdICAiNDQ6OGE6NWI6ODQ6N2Q6YjMiIDE1MDAgW10gICAgICAgICAgIm92c2Jy
aWRnZTAiIDY1NTM0ICBbXSAgICAgICAgICAgICB7fSAgICAgIHt9ICAgICAgICAgICB7Y29sbGlz
aW9ucz0wLCByeF9ieXRlcz0xOTM3MTgsIHJ4X2NyY19lcnI9MCwgcnhfZHJvcHBlZD0zOSwgcnhf
ZXJyb3JzPTAsIHJ4X2ZyYW1lX2Vycj0wLCByeF9vdmVyX2Vycj0wLCByeF9wYWNrZXRzPTI2NzIs
IHR4X2J5dGVzPTMyNzE5LCB0eF9kcm9wcGVkPTAsIHR4X2Vycm9ycz0wLCB0eF9wYWNrZXRzPTI0
OH0ge2RyaXZlcl9uYW1lPW9wZW52c3dpdGNofSAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgaW50ZXJuYWwKZDhjNTc4MGQtYjliYi00NzdhLTg3MzItYjI5YzI4
MDIwODMxIHVwICAgICAgICAgIHt9ICB7fSAgICAgICAgIFtdICAgICAgICBbXSAgICAgICAgICAg
ICAgIFtdICAgICAgICAgICAgIFtdICAgICAgICAgW10gICAgICAgW10gICAgICAgICAgICAgICBb
XSAgICAgICAgICAgICAgICAgW10gICAgIFtdICAgIHt9ICAgICAgICAgICA2ICAgICAgIDAgICAg
ICAgICAgICAgICAgICAgICAgMCAgICAgICAgICAgICAgICAgICAgIFtdICAgICAgICAgICAxICAg
ICAgICAgICBbXSAgICAgICAgIHVwICAgICAgICAge30gICBbXSAgImE2OmY2OmU1OmE0OjViOjQ1
IiAxNTAwIFtdICAgICAgICAgIG92aXJ0YnJpZGdlICA2NTUzNCAgW10gICAgICAgICAgICAge30g
ICAgICB7fSAgICAgICAgICAge2NvbGxpc2lvbnM9MCwgcnhfYnl0ZXM9MCwgcnhfY3JjX2Vycj0w
LCByeF9kcm9wcGVkPTAsIHJ4X2Vycm9ycz0wLCByeF9mcmFtZV9lcnI9MCwgcnhfb3Zlcl9lcnI9
MCwgcnhfcGFja2V0cz0wLCB0eF9ieXRlcz04MjIsIHR4X2Ryb3BwZWQ9MCwgdHhfZXJyb3JzPTAs
IHR4X3BhY2tldHM9MTN9ICAgICAgICAgICAgIHtkcml2ZXJfbmFtZT1vcGVudnN3aXRjaH0gICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGludGVybmFsCjBhYzRh
MWE0LTFmMDYtNGJhNi1iMDEzLWM0ZmI0NTRmMzE1YyB1cCAgICAgICAgICB7fSAge30gICAgICAg
ICBbXSAgICAgICAgW10gICAgICAgICAgICAgICBbXSAgICAgICAgICAgICBbXSAgICAgICAgIFtd
ICAgICAgIFtdICAgICAgICAgICAgICAgW10gICAgICAgICAgICAgICAgIGZ1bGwgICBbXSAgICB7
fSAgICAgICAgICAgMiAgICAgICAwICAgICAgICAgICAgICAgICAgICAgIDAgICAgICAgICAgICAg
ICAgICAgICBbXSAgICAgICAgICAgMSAgICAgICAgICAgMTAwMDAwMDAwMCB1cCAgICAgICAgIHt9
ICAgW10gICI0NDo4YTo1Yjo4NDo3ZDpiMyIgMTUwMCBbXSAgICAgICAgICAiZXRoMCIgICAgICAg
MSAgICAgIFtdICAgICAgICAgICAgIHt9ICAgICAge30gICAgICAgICAgIHtjb2xsaXNpb25zPTAs
IHJ4X2J5dGVzPTIzNDM5NCwgcnhfY3JjX2Vycj0wLCByeF9kcm9wcGVkPTAsIHJ4X2Vycm9ycz0w
LCByeF9mcmFtZV9lcnI9MCwgcnhfb3Zlcl9lcnI9MCwgcnhfcGFja2V0cz0yNzExLCB0eF9ieXRl
cz0zMzYwMywgdHhfZHJvcHBlZD0wLCB0eF9lcnJvcnM9MCwgdHhfcGFja2V0cz0yNTd9ICB7ZHJp
dmVyX25hbWU9InI4MTY5IiwgZHJpdmVyX3ZlcnNpb249IjIuM0xLLU5BUEkiLCBmaXJtd2FyZV92
ZXJzaW9uPSIifSAiIiAgICAgIAoKTWFuYWdlciB0YWJsZQpfdXVpZCBjb25uZWN0aW9uX21vZGUg
ZXh0ZXJuYWxfaWRzIGluYWN0aXZpdHlfcHJvYmUgaXNfY29ubmVjdGVkIG1heF9iYWNrb2ZmIG90
aGVyX2NvbmZpZyBzdGF0dXMgdGFyZ2V0Ci0tLS0tIC0tLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0t
LS0gLS0tLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0t
IC0tLS0tLSAtLS0tLS0KCk1pcnJvciB0YWJsZQpfdXVpZCBleHRlcm5hbF9pZHMgbmFtZSBvdXRw
dXRfcG9ydCBvdXRwdXRfdmxhbiBzZWxlY3RfYWxsIHNlbGVjdF9kc3RfcG9ydCBzZWxlY3Rfc3Jj
X3BvcnQgc2VsZWN0X3ZsYW4gc25hcGxlbiBzdGF0aXN0aWNzCi0tLS0tIC0tLS0tLS0tLS0tLSAt
LS0tIC0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tIC0tLS0tLS0tLS0gLS0tLS0tLS0tLS0tLS0tIC0t
LS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLSAtLS0tLS0tIC0tLS0tLS0tLS0KCk5ldEZsb3cgdGFi
bGUKX3V1aWQgYWN0aXZlX3RpbWVvdXQgYWRkX2lkX3RvX2ludGVyZmFjZSBlbmdpbmVfaWQgZW5n
aW5lX3R5cGUgZXh0ZXJuYWxfaWRzIHRhcmdldHMKLS0tLS0gLS0tLS0tLS0tLS0tLS0gLS0tLS0t
LS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0gLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0K
Ck9wZW5fdlN3aXRjaCB0YWJsZQpfdXVpZCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
YnJpZGdlcyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
IGN1cl9jZmcgZGF0YXBhdGhfdHlwZXMgICBkYl92ZXJzaW9uIGV4dGVybmFsX2lkcyAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGlmYWNlX3R5cGVzICAgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWFuYWdlcl9vcHRpb25zIG5leHRf
Y2ZnIG90aGVyX2NvbmZpZyBvdnNfdmVyc2lvbiBzc2wgc3RhdGlzdGljcyBzeXN0ZW1fdHlwZSBz
eXN0ZW1fdmVyc2lvbgotLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0gLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0t
LS0gLS0tLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tIC0t
LS0tLS0tLS0tLSAtLS0tLS0tLS0tLSAtLS0gLS0tLS0tLS0tLSAtLS0tLS0tLS0tLSAtLS0tLS0t
LS0tLS0tLQplYmI2YWVkZS1jYmJjLTRmNGYtYTg4YS1hOWNkNzJiMmJkMjMgWzlkM2FiMDllLWEx
NDYtNGJmMi1hNWJiLWJhOTk0OGM0ZjJkZCwgYTRlNWY2YTUtNGVjMS00NTViLTk4ZTktOGI1ZTZh
OGI0Y2M0LCBkNGFmNWJiYi1kZjE0LTRmZmMtYWFlOS1iNjU2NmE1ZmZkODddIDY4ICAgICAgW25l
dGRldiwgc3lzdGVtXSAiNy4xNC4wIiAgIHtob3N0bmFtZT0iaDIubGltZXRyYW5zaXQuY29tIiwg
b3ZuLWVuY2FwLWlwPSIxNzIuMjcuMS4xIiwgb3ZuLWVuY2FwLXR5cGU9Z2VuZXZlLCBvdm4tcmVt
b3RlPSJ0Y3A6MTI3LjAuMC4xOjY2NDIiLCBzeXN0ZW0taWQ9IjZlNGRkMjlmLTc2MDctNDhkNy04
ZTVhLWVlZjRjNmFlZWZiNSJ9IFtnZW5ldmUsIGdyZSwgaW50ZXJuYWwsIGxpc3AsIHBhdGNoLCBz
dHQsIHN5c3RlbSwgdGFwLCB2eGxhbl0gW10gICAgICAgICAgICAgIDY4ICAgICAgIHt9ICAgICAg
ICAgICAiMi42LjkwIiAgICBbXSAge30gICAgICAgICBjZW50b3MgICAgICAiNyIgICAgICAgICAg
IAoKUG9ydCB0YWJsZQpfdXVpZCAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYm9uZF9h
Y3RpdmVfc2xhdmUgYm9uZF9kb3duZGVsYXkgYm9uZF9mYWtlX2lmYWNlIGJvbmRfbW9kZSBib25k
X3VwZGVsYXkgZXh0ZXJuYWxfaWRzIGZha2VfYnJpZGdlIGludGVyZmFjZXMgICAgICAgICAgICAg
ICAgICAgICAgICAgICAgIGxhY3AgbWFjIG5hbWUgICAgICAgICBvdGhlcl9jb25maWcgcHJvdGVj
dGVkIHFvcyByc3RwX3N0YXRpc3RpY3MgcnN0cF9zdGF0dXMgc3RhdGlzdGljcyBzdGF0dXMgdGFn
IHRydW5rcyB2bGFuX21vZGUKLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tIC0t
LS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLS0tLSAtLS0tLS0tLS0g
LS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLSAtLS0tLS0tLS0tLSAtLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLSAtLS0tIC0tLSAtLS0tLS0tLS0tLS0gLS0tLS0tLS0tLS0tIC0t
LS0tLS0tLSAtLS0gLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tIC0tLS0tLS0tLS0gLS0tLS0t
IC0tLSAtLS0tLS0gLS0tLS0tLS0tCjQwNmNhZjcyLWE2ZjktNGZkOC04M2RjLTJiYzRmYjIxOTQ0
YyBbXSAgICAgICAgICAgICAgICAwICAgICAgICAgICAgICBmYWxzZSAgICAgICAgICAgW10gICAg
ICAgIDAgICAgICAgICAgICB7fSAgICAgICAgICAgZmFsc2UgICAgICAgW2RiNGZmYWMwLWZkOTgt
NDE0My05MWMwLTNjYTQ3NjdlYmM1Ml0gW10gICBbXSAgYnItaW50ICAgICAgIHt9ICAgICAgICAg
ICBmYWxzZSAgICAgW10gIHt9ICAgICAgICAgICAgICB7fSAgICAgICAgICB7fSAgICAgICAgIHt9
ICAgICBbXSAgW10gICAgIFtdICAgICAgIApiNzM2MWM1Ny00MWFhLTRhOGYtYjZmZS02N2U2NDMx
MjlhY2EgW10gICAgICAgICAgICAgICAgMCAgICAgICAgICAgICAgZmFsc2UgICAgICAgICAgIFtd
ICAgICAgICAwICAgICAgICAgICAge30gICAgICAgICAgIGZhbHNlICAgICAgIFswYWM0YTFhNC0x
ZjA2LTRiYTYtYjAxMy1jNGZiNDU0ZjMxNWNdIFtdICAgW10gICJldGgwIiAgICAgICB7fSAgICAg
ICAgICAgZmFsc2UgICAgIFtdICB7fSAgICAgICAgICAgICAge30gICAgICAgICAge30gICAgICAg
ICB7fSAgICAgW10gIFtdICAgICBbXSAgICAgICAKN2I0NTkxN2ItYWJhZi00ZWJkLWI1MDEtY2U3
NmYwN2ZlNjVlIFtdICAgICAgICAgICAgICAgIDAgICAgICAgICAgICAgIGZhbHNlICAgICAgICAg
ICBbXSAgICAgICAgMCAgICAgICAgICAgIHt9ICAgICAgICAgICBmYWxzZSAgICAgICBbZDhjNTc4
MGQtYjliYi00NzdhLTg3MzItYjI5YzI4MDIwODMxXSBbXSAgIFtdICBvdmlydGJyaWRnZSAge30g
ICAgICAgICAgIGZhbHNlICAgICBbXSAge30gICAgICAgICAgICAgIHt9ICAgICAgICAgIHt9ICAg
ICAgICAge30gICAgIFtdICBbXSAgICAgW10gICAgICAgCmE3Y2NhOGY1LTM0MzctNDNkYy04MzEw
LTE5NTQ1NGZiNzc3MSBbXSAgICAgICAgICAgICAgICAwICAgICAgICAgICAgICBmYWxzZSAgICAg
ICAgICAgW10gICAgICAgIDAgICAgICAgICAgICB7fSAgICAgICAgICAgZmFsc2UgICAgICAgWzIx
ZGY3YTU4LWU1NzItNDA0Mi04YjY0LTYzNzBkOGU1OGY5Ml0gW10gICBbXSAgIm92c2JyaWRnZTAi
IHt9ICAgICAgICAgICBmYWxzZSAgICAgW10gIHt9ICAgICAgICAgICAgICB7fSAgICAgICAgICB7
fSAgICAgICAgIHt9ICAgICBbXSAgW10gICAgIFtdICAgICAgIAoKUW9TIHRhYmxlCl91dWlkIGV4
dGVybmFsX2lkcyBvdGhlcl9jb25maWcgcXVldWVzIHR5cGUKLS0tLS0gLS0tLS0tLS0tLS0tIC0t
LS0tLS0tLS0tLSAtLS0tLS0gLS0tLQoKUXVldWUgdGFibGUKX3V1aWQgZHNjcCBleHRlcm5hbF9p
ZHMgb3RoZXJfY29uZmlnCi0tLS0tIC0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tLQoKU1NM
IHRhYmxlCl91dWlkIGJvb3RzdHJhcF9jYV9jZXJ0IGNhX2NlcnQgY2VydGlmaWNhdGUgZXh0ZXJu
YWxfaWRzIHByaXZhdGVfa2V5Ci0tLS0tIC0tLS0tLS0tLS0tLS0tLS0tIC0tLS0tLS0gLS0tLS0t
LS0tLS0gLS0tLS0tLS0tLS0tIC0tLS0tLS0tLS0tCgpzRmxvdyB0YWJsZQpfdXVpZCBhZ2VudCBl
eHRlcm5hbF9pZHMgaGVhZGVyIHBvbGxpbmcgc2FtcGxpbmcgdGFyZ2V0cwotLS0tLSAtLS0tLSAt
LS0tLS0tLS0tLS0gLS0tLS0tIC0tLS0tLS0gLS0tLS0tLS0gLS0tLS0tLQo=
------=_Part_5778955_901085841.1483014842363--