collect artifacts in master experimental timed out

Wed Jul 5 07:51:39 UTC 2017

On Wed, Jul 5, 2017 at 9:46 AM, Nadav Goldin <ngoldin at redhat.com> wrote:

> Hi,
> Did it happen more than once?
>

No

>
> Looking at the logs what happened was:
> 1. Lago checked the engine was SSH reachable - this was true.
> 2. Then it tried connecting via SSH and collect the logs and timed out.
>
> On (1) we have retries and guards, on (2) we don't, as we assume (1)
> just passed. I guess in some conditions that logic can be flawed. Can
> you open an issue[1]?
>

Will do

>
> I'll try to fix it as soon as possible.
>
> Thanks,
>
>
> [1] https://github.com/lago-project/lago/issues
> Nadav.
>
> On Wed, Jul 5, 2017 at 9:07 AM, Gil Shinar <gshinar at redhat.com> wrote:
> > Hi Nadav/Gal,
> >
> > I see the folowing exceptions in lago log:
> >
> > 2017-07-04 14:24:39,254::log_utils.py::__exit__::606::lago.prefix::
> DEBUG::
> > File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> > _collect_artifacts
> >     vm.collect_artifacts(path, ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624,
> in
> > collect_artifacts
> >     ignore_nopath=ignore_nopath
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381,
> in
> > extract_paths
> >     return self.provider.extract_paths(paths, *args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py",
> line
> > 297, in extract_paths
> >     ignore_nopath=ignore_nopath,
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247,
> in
> > extract_paths
> >     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266,
> in
> > _extract_paths_scp
> >     propagate_fail=False
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425,
> in
> > copy_from
> >     local_path=local_path,
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
> >     self._recv_all()
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
> >     msg = self.channel.recv(1024)
> >   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line
> 615, in
> > recv
> >     raise socket.timeout()
> >
> > 2017-07-04
> > 14:24:39,254::utils.py::_ret_via_queue::60::lago.utils::ERROR::Error
> while
> > running thread
> > Traceback (most recent call last):
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in
> > _ret_via_queue
> >     queue.put({'return': func()})
> >   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> > _collect_artifacts
> >     vm.collect_artifacts(path, ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624,
> in
> > collect_artifacts
> >     ignore_nopath=ignore_nopath
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381,
> in
> > extract_paths
> >     return self.provider.extract_paths(paths, *args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py",
> line
> > 297, in extract_paths
> >     ignore_nopath=ignore_nopath,
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247,
> in
> > extract_paths
> >     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266,
> in
> > _extract_paths_scp
> >     propagate_fail=False
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425,
> in
> > copy_from
> >     local_path=local_path,
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
> >     self._recv_all()
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
> >     msg = self.channel.recv(1024)
> >   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line
> 615, in
> > recv
> >     raise socket.timeout()
> > timeout
> > 2017-07-04 14:24:39,255::log_utils.py::end_log_task::669::root::ERROR::@
> > Collect artifacts:  [31mERROR [0m (in 0:00:05)
> > 2017-07-04 14:24:39,256::log_utils.py::__exit__::606::lago.prefix::
> DEBUG::
> > File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 635, in
> > wrapper
> >     return func(*args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1480, in
> > collect_artifacts
> >     self.virt_env.get_vms().values(),
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in
> > invoke_in_parallel
> >     return vt.join_all()
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in
> > _ret_via_queue
> >     queue.put({'return': func()})
> >   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> > _collect_artifacts
> >     vm.collect_artifacts(path, ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624,
> in
> > collect_artifacts
> >     ignore_nopath=ignore_nopath
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381,
> in
> > extract_paths
> >     return self.provider.extract_paths(paths, *args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py",
> line
> > 297, in extract_paths
> >     ignore_nopath=ignore_nopath,
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247,
> in
> > extract_paths
> >     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266,
> in
> > _extract_paths_scp
> >     propagate_fail=False
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425,
> in
> > copy_from
> >     local_path=local_path,
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
> >     self._recv_all()
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
> >     msg = self.channel.recv(1024)
> >   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line
> 615, in
> > recv
> >     raise socket.timeout()
> >
> > 2017-07-04 14:24:39,256::cmd.py::main::960::cli::ERROR::Error occured,
> > aborting
> > Traceback (most recent call last):
> >   File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 954, in main
> >     cli_plugins[args.verb].do_run(args)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/cli.py", line
> 184, in
> > do_run
> >     self._do_run(**vars(args))
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 501, in
> > wrapper
> >     return func(*args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 512, in
> > wrapper
> >     return func(*args, prefix=prefix, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 751, in
> > do_collect
> >     prefix.collect_artifacts(output, ignore_nopath=not no_skip)
> >   File "/usr/lib/python2.7/site-packages/lago/sdk_utils.py", line 50, in
> > wrapped
> >     return func(*args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 635,
> in
> > wrapper
> >     return func(*args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1480, in
> > collect_artifacts
> >     self.virt_env.get_vms().values(),
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in
> > invoke_in_parallel
> >     return vt.join_all()
> >   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in
> > _ret_via_queue
> >     queue.put({'return': func()})
> >   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> > _collect_artifacts
> >     vm.collect_artifacts(path, ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624,
> in
> > collect_artifacts
> >     ignore_nopath=ignore_nopath
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381,
> in
> > extract_paths
> >     return self.provider.extract_paths(paths, *args, **kwargs)
> >   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py",
> line
> > 297, in extract_paths
> >     ignore_nopath=ignore_nopath,
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247,
> in
> > extract_paths
> >     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266,
> in
> > _extract_paths_scp
> >     propagate_fail=False
> >   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425,
> in
> > copy_from
> >     local_path=local_path,
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
> >     self._recv_all()
> >   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
> >     msg = self.channel.recv(1024)
> >   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line
> 615, in
> > recv
> >     raise socket.timeout()
> > timeout
> >
> >
> > Link to build:
> > http://jenkins.ovirt.org/job/test-repo_ovirt_experimental_master/7459
> >
> > Thanks
> > Gil
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.ovirt.org/pipermail/infra/attachments/20170705/9f4da258/attachment.html>