<div dir="ltr"><br><div class="gmail_extra"><br><div class="gmail_quote">On Fri, Nov 25, 2016 at 7:52 AM, knarra <span dir="ltr">&lt;<a href="mailto:knarra@redhat.com" target="_blank">knarra@redhat.com</a>&gt;</span> wrote:<br><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
  
    
  
  <div bgcolor="#FFFFFF"><div><div class="gmail-h5">
    <div class="gmail-m_-1140308026537364043moz-cite-prefix">On 11/25/2016 03:35 AM, Simone
      Tiraboschi wrote:<br>
    </div>
    <blockquote type="cite">
      <div dir="ltr">
        <div class="gmail_extra"><br>
          <div class="gmail_quote">On Thu, Nov 24, 2016 at 3:35 PM,
            Simone Tiraboschi <span dir="ltr">&lt;<a href="mailto:stirabos@redhat.com" target="_blank">stirabos@redhat.com</a>&gt;</span>
            wrote:<br>
            <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
              <div dir="ltr"><br>
                <div class="gmail_extra"><br>
                  <div class="gmail_quote">
                    <div>
                      <div class="gmail-m_-1140308026537364043gmail-h5">On Thu, Nov 24, 2016 at 3:23
                        PM, knarra <span dir="ltr">&lt;<a href="mailto:knarra@redhat.com" target="_blank">knarra@redhat.com</a>&gt;</span>
                        wrote:<br>
                        <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                          <div bgcolor="#FFFFFF">
                            <div>
                              <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-h5">
                                <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993moz-cite-prefix">On
                                  11/24/2016 07:47 PM, Simone Tiraboschi
                                  wrote:<br>
                                </div>
                                <blockquote type="cite">
                                  <div dir="ltr"><br>
                                    <div class="gmail_extra"><br>
                                      <div class="gmail_quote">On Thu,
                                        Nov 24, 2016 at 3:06 PM, knarra
                                        <span dir="ltr">&lt;<a href="mailto:knarra@redhat.com" target="_blank">knarra@redhat.com</a>&gt;</span>
                                        wrote:<br>
                                        <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                          <div bgcolor="#FFFFFF">
                                            <div>
                                              <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-h5">
                                                <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601moz-cite-prefix">On
                                                  11/24/2016 07:27 PM,
                                                  Simone Tiraboschi
                                                  wrote:<br>
                                                </div>
                                                <blockquote type="cite">
                                                  <div dir="ltr"><br>
                                                    <div class="gmail_extra"><br>
                                                      <div class="gmail_quote">On
                                                        Thu, Nov 24,
                                                        2016 at 2:39 PM,
                                                        knarra <span dir="ltr">&lt;<a href="mailto:knarra@redhat.com" target="_blank">knarra@redhat.com</a>&gt;</span>
                                                        wrote:<br>
                                                        <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                                          <div bgcolor="#FFFFFF"><span class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-">
                                                          <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-m_-2672125017527102966moz-cite-prefix">On
                                                          11/24/2016
                                                          06:56 PM,
                                                          Simone
                                                          Tiraboschi
                                                          wrote:<br>
                                                          </div>
                                                          <blockquote type="cite">
                                                          <div dir="ltr"><br>
                                                          <div class="gmail_extra"><br>
                                                          <div class="gmail_quote">On
                                                          Thu, Nov 24,
                                                          2016 at 2:08
                                                          PM, knarra <span dir="ltr">&lt;<a href="mailto:knarra@redhat.com" target="_blank">knarra@redhat.com</a>&gt;</span>
                                                          wrote:<br>
                                                          <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                                          <div bgcolor="#FFFFFF">
                                                          <div>
                                                          <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-m_-2672125017527102966h5">
                                                          <div class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-m_-2672125017527102966m_-6170493833456119351moz-cite-prefix">On
                                                          11/24/2016
                                                          06:15 PM,
                                                          Simone
                                                          Tiraboschi
                                                          wrote:<br>
                                                          </div>
                                                          <blockquote type="cite">
                                                          <div dir="ltr"><br>
                                                          <div class="gmail_extra"><br>
                                                          <div class="gmail_quote">On
                                                          Thu, Nov 24,
                                                          2016 at 1:26
                                                          PM, knarra <span dir="ltr">&lt;<a href="mailto:knarra@redhat.com" target="_blank">knarra@redhat.com</a>&gt;</span>
                                                          wrote:<br>
                                                          <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Hi,<br>
                                                          <br>
                                                              I have
                                                          three nodes
                                                          with glusterfs
                                                          as storage
                                                          domain. For
                                                          some reason i
                                                          see that
                                                          vm.conf from
                                                          /var/run/ovirt-hosted-engine-h<wbr>a
                                                          is missing and
                                                          due to this on
                                                          one of my host
                                                          i see that
                                                          Hosted Engine
                                                          HA : Not
                                                          Active. Once i
                                                          copy the file
                                                          from some
                                                          other node and
                                                          restart
                                                          ovirt-ha-broker
                                                          and
                                                          ovirt-ha-agent
                                                          services
                                                          everything
                                                          works fine.
                                                          But then this
                                                          happens again.
                                                          Can some
                                                          please help me
                                                          identify why
                                                          this happens.
                                                          Below is the
                                                          log i see in
                                                          ovirt-ha-agent.logs.<br>
                                                          <br>
                                                          <br>
                                                          <a href="https://paste.fedoraproject.org/489120/79990345/" rel="noreferrer" target="_blank">https://paste.fedoraproject.or<wbr>g/489120/79990345/</a><br>
                                                          <br>
                                                          </blockquote>
                                                          <div><br>
                                                          </div>
                                                          Once the
                                                          engine
                                                          correctly
                                                          imported the
                                                          hosted-engine
                                                          storage
                                                          domain, a
                                                          couple of
                                                          OVF_STORE
                                                          volumes will
                                                          appear there.<br>
                                                          Every
                                                          modification
                                                          to the engine
                                                          VM
                                                          configuration
                                                          will be
                                                          written by the
                                                          engine into
                                                          that
                                                          OVF_STORE, so
                                                          all the
                                                          ovirt-ha-agent
                                                          running on the
                                                          hosted-engine
                                                          hosts will be
                                                          able to
                                                          re-start the
                                                          engine VM with
                                                          a coherent
                                                          configuration.<br>
                                                          <br>
                                                          Till the
                                                          engine imports
                                                          the
                                                          hosted-engine
                                                          storage
                                                          domain,
                                                          ovirt-ha-agent
                                                          will fall back
                                                          to the initial
                                                          vm.conf.<br>
                                                          <br>
                                                          In you case
                                                          the OVF_STORE
                                                          volume is
                                                          there,<br>
                                                          but the agent
                                                          fails
                                                          extracting the
                                                          engine VM
                                                          configuration:<br>
                                                          MainThread::<a class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-m_-2672125017527102966m_-6170493833456119351moz-txt-link-freetext">INFO::2016-11-24</a>
17:55:04,914::ovf_store::112::<wbr>ovirt_hosted_engine_ha.lib.ovf<wbr>.ovf_store.OVFStore::(getEngin<wbr>eVMOVF)
                                                          Extracting
                                                          Engine VM OVF
                                                          from the
                                                          OVF_STORE<br>
                                                          MainThread::<a class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-m_-2672125017527102966m_-6170493833456119351moz-txt-link-freetext">INFO::2016-11-24</a>
17:55:04,919::ovf_store::119::<wbr>ovirt_hosted_engine_ha.lib.ovf<wbr>.ovf_store.OVFStore::(getEngin<wbr>eVMOVF)
                                                          OVF_STORE
                                                          volume path:
                                                          /rhev/data-center/mnt/glusterS<wbr>D/10.70.36.79:_engine/27f054c3<wbr>-c245-4039-b42a-c28b37043016/i<wbr>mages/fdf49778-9a06-49c6-bf7a-<wbr>a0f12425911c/8c954add-6bcf-47f<wbr>8-ac2e-4c85fc3f8699<br>
MainThread::ERROR::2016-11-24 17:55:04,928::ovf_store::124::<wbr>ovirt_hosted_engine_ha.lib.ovf<wbr>.ovf_store.OVFStore::(getEngin<wbr>eVMOVF)
                                                          Unable to
                                                          extract HEVM
                                                          OVF<br>
                                                          <br>
                                                          So it tries to
                                                          rollback to
                                                          the initial
                                                          vm.conf, but
                                                          also that one
                                                          seams to miss
                                                          some values
                                                          and so the
                                                          agent is
                                                          failing:<br>
MainThread::ERROR::2016-11-24 17:55:04,974::agent::205::ovir<wbr>t_hosted_engine_ha.agent.agent<wbr>.Agent::(_run_agent)
                                                          Error:
                                                          &#39;&#39;Configuration
                                                          value not
                                                          found:
                                                          file=/var/run/ovirt-hosted-eng<wbr>ine-ha/vm.conf,
                                                          key=memSize&#39;&#39;
                                                          - trying to
                                                          restart agent<br>
                                                          <br>
                                                          Both of the
                                                          issue seams
                                                          storage
                                                          related, could
                                                          yuo please
                                                          share your
                                                          gluster logs?<br>
                                                          <br>
                                                           
                                                          <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                                          <br>
                                                          Thanks<br>
                                                          <br>
                                                          kasturi<br>
                                                          <br>
                                                          </blockquote>
                                                          </div>
                                                          <br>
                                                          </div>
                                                          </div>
                                                          </blockquote>
                                                          </div>
                                                          </div>
                                                          <p>Hi Simone,</p>
                                                          <p>    Below
                                                          [1] is the
                                                          link for the
                                                          sosreports on
                                                          the first two
                                                          hosts. The
                                                          third host has
                                                          some issue.
                                                          Once it is up
                                                          will give the
                                                          sosreport from
                                                          there as well.</p>
                                                          </div>
                                                          </blockquote>
                                                          <div><br>
                                                          </div>
                                                          <div>And the
                                                          host where you
                                                          see the
                                                          initial issue
                                                          was the third
                                                          one? <br>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </blockquote>
                                                          </span> It is
                                                          on the first
                                                          host.<span class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-"><br>
                                                          <blockquote type="cite">
                                                          <div dir="ltr">
                                                          <div class="gmail_extra">
                                                          <div class="gmail_quote">
                                                          <div> </div>
                                                          </div>
                                                          </div>
                                                          </div>
                                                          </blockquote>
                                                          </span></div>
                                                        </blockquote>
                                                        <div><br>
                                                        </div>
                                                        <div>It seams
                                                          that host1 is
                                                          failing
                                                          reading from
                                                          the the
                                                          hosted-engine
                                                          storage
                                                          domain:</div>
                                                        <div><br>
                                                        </div>
                                                        <div>
                                                          <div>[2016-11-24
12:33:43.678467] W [MSGID: 114031] [client-rpc-fops.c:2938:client<wbr>3_3_lookup_cbk]
0-engine-client-2: remote operation failed. Path: /
                                                          (00000000-0000-0000-0000-00000<wbr>0000001)
                                                          [Transport
                                                          endpoint is
                                                          not connected]</div>
                                                          <div>[2016-11-24
12:33:43.678747] E [rpc-clnt.c:365:saved_frames_u<wbr>nwind] (--&gt;
                                                          /lib64/libglusterfs.so.0(_gf_l<wbr>og_callingfn+0x192)[0x7f077eba<wbr>1642]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_unwind+0x1de)[0x7f077e9677<wbr>5e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_destroy+0xe)[0x7f077e96786<wbr>e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>connection_cleanup+0x84)[0x7f0<wbr>77e968fc4]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>notify+0x120)[0x7f077e9698a0]
                                                          )))))
                                                          0-engine-client-2:
                                                          forced
                                                          unwinding
                                                          frame
                                                          type(GlusterFS
                                                          3.3)
                                                          op(LOOKUP(27))
                                                          called at
                                                          2016-11-24
                                                          12:33:07.495178
                                                          (xid=0x82a1c)</div>
                                                          <div>[2016-11-24
12:33:43.678982] E [rpc-clnt.c:365:saved_frames_u<wbr>nwind] (--&gt;
                                                          /lib64/libglusterfs.so.0(_gf_l<wbr>og_callingfn+0x192)[0x7f077eba<wbr>1642]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_unwind+0x1de)[0x7f077e9677<wbr>5e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_destroy+0xe)[0x7f077e96786<wbr>e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>connection_cleanup+0x84)[0x7f0<wbr>77e968fc4]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>notify+0x120)[0x7f077e9698a0]
                                                          )))))
                                                          0-engine-client-2:
                                                          forced
                                                          unwinding
                                                          frame
                                                          type(GlusterFS
                                                          3.3)
                                                          op(LOOKUP(27))
                                                          called at
                                                          2016-11-24
                                                          12:33:08.770637
                                                          (xid=0x82a1d)</div>
                                                          <div>[2016-11-24
12:33:43.679001] W [MSGID: 114031] [client-rpc-fops.c:2938:client<wbr>3_3_lookup_cbk]
0-engine-client-2: remote operation failed. Path:
                                                          /27f054c3-c245-4039-b42a-c28b3<wbr>7043016/images/39960f40-4aae-4<wbr>714-ba73-1637785fae7c/38fa3519<wbr>-f21e-4671-8c69-d1497ff8a490
(1090c25b-9c90-434e-a133-faf96<wbr>47cc992) [Transport endpoint is not
                                                          connected]</div>
                                                          <div>[2016-11-24
12:33:43.679303] E [rpc-clnt.c:365:saved_frames_u<wbr>nwind] (--&gt;
                                                          /lib64/libglusterfs.so.0(_gf_l<wbr>og_callingfn+0x192)[0x7f077eba<wbr>1642]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_unwind+0x1de)[0x7f077e9677<wbr>5e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_destroy+0xe)[0x7f077e96786<wbr>e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>connection_cleanup+0x84)[0x7f0<wbr>77e968fc4]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>notify+0x120)[0x7f077e9698a0]
                                                          )))))
                                                          0-engine-client-2:
                                                          forced
                                                          unwinding
                                                          frame
                                                          type(GlusterFS
                                                          3.3)
                                                          op(LOOKUP(27))
                                                          called at
                                                          2016-11-24
                                                          12:33:11.096856
                                                          (xid=0x82a1e)</div>
                                                          <div>[2016-11-24
12:33:43.679596] E [rpc-clnt.c:365:saved_frames_u<wbr>nwind] (--&gt;
                                                          /lib64/libglusterfs.so.0(_gf_l<wbr>og_callingfn+0x192)[0x7f077eba<wbr>1642]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_unwind+0x1de)[0x7f077e9677<wbr>5e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(saved_fra<wbr>mes_destroy+0xe)[0x7f077e96786<wbr>e]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>connection_cleanup+0x84)[0x7f0<wbr>77e968fc4]
                                                          (--&gt;
                                                          /lib64/libgfrpc.so.0(rpc_clnt_<wbr>notify+0x120)[0x7f077e9698a0]
                                                          )))))
                                                          0-engine-client-2:
                                                          forced
                                                          unwinding
                                                          frame
                                                          type(GF-DUMP)
                                                          op(NULL(2))
                                                          called at
                                                          2016-11-24
                                                          12:33:13.673743
                                                          (xid=0x82a1f)</div>
                                                          <div>[2016-11-24
12:33:43.682310] I [socket.c:3401:socket_submit_r<wbr>equest]
                                                          0-engine-client-2:
                                                          not connected
(priv-&gt;connected = 0)</div>
                                                          <div>[2016-11-24
12:33:43.682328] W [rpc-clnt.c:1640:rpc_clnt_subm<wbr>it]
                                                          0-engine-client-2:
                                                          failed to
                                                          submit
                                                          rpc-request
                                                          (XID: 0x82a20
                                                          Program:
                                                          GlusterFS 3.3,
                                                          ProgVers: 330,
                                                          Proc: 30) to
                                                          rpc-transport
(engine-client-2)</div>
                                                          <div>[2016-11-24
12:33:43.682391] W [rpc-clnt.c:1640:rpc_clnt_subm<wbr>it]
                                                          0-engine-client-2:
                                                          failed to
                                                          submit
                                                          rpc-request
                                                          (XID: 0x82a21
                                                          Program:
                                                          GlusterFS 3.3,
                                                          ProgVers: 330,
                                                          Proc: 27) to
                                                          rpc-transport
(engine-client-2)</div>
                                                          <div>[2016-11-24
12:33:43.682441] W [rpc-clnt.c:1640:rpc_clnt_subm<wbr>it]
                                                          0-engine-client-2:
                                                          failed to
                                                          submit
                                                          rpc-request
                                                          (XID: 0x82a22
                                                          Program:
                                                          GlusterFS 3.3,
                                                          ProgVers: 330,
                                                          Proc: 27) to
                                                          rpc-transport
(engine-client-2)</div>
                                                          <div>[2016-11-24
12:33:43.682441] W [MSGID: 114031] [client-rpc-fops.c:2938:client<wbr>3_3_lookup_cbk]
0-engine-client-2: remote operation failed. Path:
                                                          /27f054c3-c245-4039-b42a-c28b3<wbr>7043016
(a64398f5-3fa3-48fe-9d40-d3860<wbr>876cc2c) [Transport endpoint is not
                                                          connected]</div>
                                                          <div>[2016-11-24
12:33:43.682492] W [rpc-clnt-ping.c:203:rpc_clnt_<wbr>ping_cbk]
                                                          0-engine-client-2:
                                                          socket
                                                          disconnected</div>
                                                          <div>[2016-11-24
12:33:43.682536] I [MSGID: 114018] [client.c:2280:client_rpc_noti<wbr>fy]
0-engine-client-2: disconnected from engine-client-2. Client process
                                                          will keep
                                                          trying to
                                                          connect to
                                                          glusterd until
                                                          brick&#39;s port
                                                          is available</div>
                                                          <div>[2016-11-24
12:33:43.682562] W [rpc-clnt.c:1640:rpc_clnt_subm<wbr>it]
                                                          0-engine-client-2:
                                                          failed to
                                                          submit
                                                          rpc-request
                                                          (XID: 0x82a23
                                                          Program:
                                                          GlusterFS 3.3,
                                                          ProgVers: 330,
                                                          Proc: 27) to
                                                          rpc-transport
(engine-client-2)</div>
                                                          <div>The
                                                          message &quot;W
                                                          [MSGID:
                                                          114031]
                                                          [client-rpc-fops.c:2938:client<wbr>3_3_lookup_cbk]
0-engine-client-2: remote operation failed. Path:
                                                          /27f054c3-c245-4039-b42a-c28b3<wbr>7043016
(a64398f5-3fa3-48fe-9d40-d3860<wbr>876cc2c) [Transport endpoint is not
                                                          connected]&quot;
                                                          repeated 2
                                                          times between
                                                          [2016-11-24
                                                          12:33:43.682441]
                                                          and
                                                          [2016-11-24
                                                          12:33:43.682599]</div>
                                                          <div>[2016-11-24
12:33:43.688324] W [MSGID: 114031] [client-rpc-fops.c:2938:client<wbr>3_3_lookup_cbk]
0-engine-client-2: remote operation failed. Path: (null)
                                                          (00000000-0000-0000-0000-00000<wbr>0000000)
                                                          [Transport
                                                          endpoint is
                                                          not connected]</div>
                                                        </div>
                                                        <div><br>
                                                        </div>
                                                        <div>Before that
                                                          there was a
                                                          lot of
                                                          self-healing
                                                          activities.</div>
                                                        <div><br>
                                                        </div>
                                                      </div>
                                                    </div>
                                                  </div>
                                                </blockquote>
                                              </div>
                                            </div>
                                            simone, these logs indicates
                                            that first host is not able
                                            to connect to the brick in
                                            the third host since it was
                                            powered down.  It reads
                                            remote operation failed on
                                            engine_client_2 .</div>
                                        </blockquote>
                                        <div><br>
                                        </div>
                                        <div>Can you please share the
                                          output of</div>
                                         source
                                        /etc/ovirt-hosted-engine/hoste<wbr>d-engine.conf<br>
                                         find /rhev/data-center/ -path
                                        &quot;*/${sdUUID}/images/${conf_ima<wbr>ge_UUID}/${conf_volume_UUID}&quot;
                                        -type f -exec sh -c &#39;sudo -u
                                        vdsm dd if=$1 2&gt;/dev/null |
                                        tar -xOvf - vm.conf
                                         2&gt;/dev/null&#39; {} {} \;
                                        <div>executed on your first
                                          host?</div>
                                      </div>
                                    </div>
                                  </div>
                                </blockquote>
                              </div>
                            </div>
                            output for the above command <a class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993moz-txt-link-freetext" href="https://paste.fedoraproject.org/489159/97176147/" target="_blank">https://paste.fedoraproject.or<wbr>g/489159/97176147/</a><br>
                            <br>
                          </div>
                        </blockquote>
                        <div><br>
                        </div>
                      </div>
                    </div>
                    <div>This one looks fine,</div>
                    <div>now the point is why the OVF_STORE doesn&#39;t
                      seams so,</div>
                    <div><br>
                    </div>
                    <div>Could you please try executing:</div>
                    <div>sudo -u vdsm dd if=<span style="font-size:12.8px">/rhev/data-center/mnt/</span><span style="font-size:12.8px">glust<wbr>erSD/10.70.36.79:_engine/</span><span style="font-size:12.8px">27f05<wbr>4c3-c245-4039-b42a-</span><span style="font-size:12.8px">c28b3704301<wbr>6/images/fdf49778-</span><span style="font-size:12.8px">9a06-49c6-bf<wbr>7a-a0f12425911c/</span><span style="font-size:12.8px">8c954add-6bcf-<wbr>47f8-ac2e-</span><span style="font-size:12.8px">4c85fc3f8699</span> |
                      tar -xOvf - 21e0e248-19bf-47b3-b72f-6a3740<wbr>d9ff43.ovf<br>
                    </div>
                    <div>
                      <div class="gmail-m_-1140308026537364043gmail-h5">
                        <div> </div>
                      </div>
                    </div>
                  </div>
                </div>
              </div>
            </blockquote>
            <div><br>
            </div>
            <div>OK, found: it&#39;s definitively a regression introduced
              (just on master) with <a href="https://gerrit.ovirt.org/#/c/66103/" target="_blank">https://gerrit.ovirt.org/#/c/<wbr>66103/</a></div>
            <div><br>
            </div>
            <div>The code that read and writes the HE configuration and
              the OVF_STORE from/to the volume on shared storage (with
              dd) is shared between ovirt-hosted-engine-setup (which
              runs as root) and ovirt-ha-agent (which runs as vdsm
              user).</div>
            <div>In order to behave correctly (against root squash on
              NFC for instance), it was always running &#39;sudo -u vdsm dd
              ...&#39; instead of just &#39;dd&#39; regardless of the active user.</div>
            <div>But since  <a href="https://gerrit.ovirt.org/#/c/66103/" target="_blank">https://gerrit.ovirt.org/#/c/<wbr>66103/</a>
              , vdsm user cannot run anymore &#39;sudo -u vdsm dd ...&#39; and
              so this issue.</div>
            <div><br>
            </div>
            <div>I opened a bug:</div>
            <div><a href="https://bugzilla.redhat.com/show_bug.cgi?id=1398443" target="_blank">https://bugzilla.redhat.com/<wbr>show_bug.cgi?id=1398443</a><br>
            </div>
          </div>
        </div>
      </div>
    </blockquote></div></div>
    Is it the same reason why vm.conf file is missing from
    /var/run/ovirt-hosted-engine-<wbr>ha folder ?</div></blockquote><div><br></div><div>/var/run/ovirt-hosted-engine-ha just just a temporary location on the host where it saves the file extracted from the OVF_STORE.</div><div>The temporary file in not that since it fails reading the OVF_STORE and the initial vm.conf due to the missing sudoers rule. <br></div><div> </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><div bgcolor="#FFFFFF"><div><div class="gmail-h5"><br>
    <blockquote type="cite">
      <div dir="ltr">
        <div class="gmail_extra">
          <div class="gmail_quote">
            <div><br>
            </div>
            <div>Kasturi, you got it just on one of your hosts since the
              other two run an older VDSM build.<br>
            </div>
            <div><br>
            </div>
            <div> </div>
            <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
              <div dir="ltr">
                <div class="gmail_extra">
                  <div class="gmail_quote">
                    <div>
                      <div class="gmail-m_-1140308026537364043gmail-h5">
                        <div><br>
                        </div>
                        <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                          <div bgcolor="#FFFFFF"> Simone, i ran the
                            following steps and i see that vm.conf file
                            is missing.<br>
                            <br>
                            1) Installed three nodes with glusterfs as
                            storage domains.<br>
                            <br>
                            2) Configured power management on all the
                            nodes and ran the command &#39;poweroff -f&#39; on
                            one of the node.<br>
                            <br>
                            3) Engine tried to fence the host and it
                            logged the message &#39;Host is rebooting&#39; but
                            this never came up.<br>
                            <br>
                            4) After some time i manually powered on the
                            host and once the host is up i see that
                            Hosted Engine HA : Not active.<br>
                            <br>
                            5) I see the following error message in
                            broker.log and vm.conf file is missing.<br>
                            <br>
                            Thread-1::ERROR::2016-11-24
                            19:32:11,651::listener::192::o<wbr>virt_hosted_engine_ha.broker.l<wbr>istener.ConnectionHandler::(ha<wbr>ndle)
                            Error handling request, data:
                            &#39;set-storage-domain<br>
                             FilesystemBackend dom_type=glusterfs
                            sd_uuid=27f054c3-c245-4039-b42<wbr>a-c28b37043016&#39;<br>
                            Traceback (most recent call last):<br>
                              File
                            &quot;/usr/lib/python2.7/site-packa<wbr>ges/ovirt_hosted_engine_ha/bro<wbr>ker/listener.py&quot;,
                            line 166, in handle<br>
                                data)<br>
                              File
                            &quot;/usr/lib/python2.7/site-packa<wbr>ges/ovirt_hosted_engine_ha/bro<wbr>ker/listener.py&quot;,
                            line 299, in _dispatch<br>
                                .set_storage_domain(client, sd_type,
                            **options)<br>
                              File
                            &quot;/usr/lib/python2.7/site-packa<wbr>ges/ovirt_hosted_engine_ha/bro<wbr>ker/storage_broker.py&quot;,
                            line 66, in set_storage_domain<br>
                                self._backends[client].connect<wbr>()<br>
                              File
                            &quot;/usr/lib/python2.7/site-packa<wbr>ges/ovirt_hosted_engine_ha/lib<wbr>/storage_backends.py&quot;,
                            line 461, in connect<br>
                                self._dom_type)<br>
                              File
                            &quot;/usr/lib/python2.7/site-packa<wbr>ges/ovirt_hosted_engine_ha/lib<wbr>/storage_backends.py&quot;,
                            line 108, in get_domain_path<br>
                                &quot; in {1}&quot;.format(sd_uuid, parent))<br>
                            BackendFailureException: path to storage
                            domain 27f054c3-c245-4039-b42a-c28b37<wbr>043016
                            not found in /rhev/data-center/mnt/glusterS<wbr>D<br>
                            <br>
                            Thanks<br>
                            kasturi<span class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-"><br>
                              <br>
                              <blockquote type="cite">
                                <div dir="ltr">
                                  <div class="gmail_extra">
                                    <div class="gmail_quote">
                                      <div><br>
                                      </div>
                                      <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                        <div bgcolor="#FFFFFF"><span class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-"><br>
                                            <blockquote type="cite">
                                              <div dir="ltr">
                                                <div class="gmail_extra">
                                                  <div class="gmail_quote">
                                                    <div><br>
                                                    </div>
                                                    <div> </div>
                                                    <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                                      <div bgcolor="#FFFFFF"><span class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-">
                                                          <blockquote type="cite">
                                                          <div dir="ltr">
                                                          <div class="gmail_extra">
                                                          <div class="gmail_quote">
                                                          <blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
                                                          <div bgcolor="#FFFFFF">
                                                          <p>[1] <a class="gmail-m_-1140308026537364043gmail-m_8283562645140910209gmail-m_4061716222045576993gmail-m_-1038946535864355601gmail-m_-2672125017527102966m_-6170493833456119351moz-txt-link-freetext" href="http://rhsqe-repo.lab.eng.blr.redhat.com/sosreports/HC/vm_conf/" target="_blank">http://rhsqe-repo.lab.eng.blr.<wbr>redhat.com/sosreports/HC/vm_co<wbr>nf/</a></p>
                                                          <p>Thanks</p>
                                                          <p>kasturi<br>
                                                          </p>
                                                          </div>
                                                          </blockquote>
                                                          </div>
                                                          <br>
                                                          </div>
                                                          </div>
                                                          </blockquote>
                                                          <p><br>
                                                          </p>
                                                        </span></div>
                                                    </blockquote>
                                                  </div>
                                                  <br>
                                                </div>
                                              </div>
                                            </blockquote>
                                            <p><br>
                                            </p>
                                          </span></div>
                                      </blockquote>
                                    </div>
                                    <br>
                                  </div>
                                </div>
                              </blockquote>
                              <p><br>
                              </p>
                            </span></div>
                        </blockquote>
                      </div>
                    </div>
                  </div>
                  <br>
                </div>
              </div>
            </blockquote>
          </div>
          <br>
        </div>
      </div>
    </blockquote>
    <p><br>
    </p>
  </div></div></div>

</blockquote></div><br></div></div>