collect artifacts in master experimental timed out

[Nadav Goldin]

Hi,
Did it happen more than once?

Looking at the logs what happened was:
1. Lago checked the engine was SSH reachable - this was true.
2. Then it tried connecting via SSH and collect the logs and timed out.

On (1) we have retries and guards, on (2) we don't, as we assume (1)
just passed. I guess in some conditions that logic can be flawed. Can
you open an issue[1]?

I'll try to fix it as soon as possible.

Thanks,


[1] https://github.com/lago-project/lago/issues
Nadav.

On Wed, Jul 5, 2017 at 9:07 AM, Gil Shinar <gshinar at redhat.com> wrote:
> Hi Nadav/Gal,
>
> I see the folowing exceptions in lago log:
>
> 2017-07-04 14:24:39,254::log_utils.py::__exit__::606::lago.prefix::DEBUG::
> File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> _collect_artifacts
>     vm.collect_artifacts(path, ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, in
> collect_artifacts
>     ignore_nopath=ignore_nopath
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, in
> extract_paths
>     return self.provider.extract_paths(paths, *args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", line
> 297, in extract_paths
>     ignore_nopath=ignore_nopath,
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, in
> extract_paths
>     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, in
> _extract_paths_scp
>     propagate_fail=False
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, in
> copy_from
>     local_path=local_path,
>   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
>     self._recv_all()
>   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
>     msg = self.channel.recv(1024)
>   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line 615, in
> recv
>     raise socket.timeout()
>
> 2017-07-04
> 14:24:39,254::utils.py::_ret_via_queue::60::lago.utils::ERROR::Error while
> running thread
> Traceback (most recent call last):
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in
> _ret_via_queue
>     queue.put({'return': func()})
>   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> _collect_artifacts
>     vm.collect_artifacts(path, ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, in
> collect_artifacts
>     ignore_nopath=ignore_nopath
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, in
> extract_paths
>     return self.provider.extract_paths(paths, *args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", line
> 297, in extract_paths
>     ignore_nopath=ignore_nopath,
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, in
> extract_paths
>     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, in
> _extract_paths_scp
>     propagate_fail=False
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, in
> copy_from
>     local_path=local_path,
>   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
>     self._recv_all()
>   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
>     msg = self.channel.recv(1024)
>   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line 615, in
> recv
>     raise socket.timeout()
> timeout
> 2017-07-04 14:24:39,255::log_utils.py::end_log_task::669::root::ERROR::@
> Collect artifacts:  [31mERROR [0m (in 0:00:05)
> 2017-07-04 14:24:39,256::log_utils.py::__exit__::606::lago.prefix::DEBUG::
> File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 635, in
> wrapper
>     return func(*args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1480, in
> collect_artifacts
>     self.virt_env.get_vms().values(),
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in
> invoke_in_parallel
>     return vt.join_all()
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in
> _ret_via_queue
>     queue.put({'return': func()})
>   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> _collect_artifacts
>     vm.collect_artifacts(path, ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, in
> collect_artifacts
>     ignore_nopath=ignore_nopath
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, in
> extract_paths
>     return self.provider.extract_paths(paths, *args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", line
> 297, in extract_paths
>     ignore_nopath=ignore_nopath,
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, in
> extract_paths
>     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, in
> _extract_paths_scp
>     propagate_fail=False
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, in
> copy_from
>     local_path=local_path,
>   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
>     self._recv_all()
>   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
>     msg = self.channel.recv(1024)
>   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line 615, in
> recv
>     raise socket.timeout()
>
> 2017-07-04 14:24:39,256::cmd.py::main::960::cli::ERROR::Error occured,
> aborting
> Traceback (most recent call last):
>   File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 954, in main
>     cli_plugins[args.verb].do_run(args)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/cli.py", line 184, in
> do_run
>     self._do_run(**vars(args))
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 501, in
> wrapper
>     return func(*args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 512, in
> wrapper
>     return func(*args, prefix=prefix, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/cmd.py", line 751, in
> do_collect
>     prefix.collect_artifacts(output, ignore_nopath=not no_skip)
>   File "/usr/lib/python2.7/site-packages/lago/sdk_utils.py", line 50, in
> wrapped
>     return func(*args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/log_utils.py", line 635, in
> wrapper
>     return func(*args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1480, in
> collect_artifacts
>     self.virt_env.get_vms().values(),
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 100, in
> invoke_in_parallel
>     return vt.join_all()
>   File "/usr/lib/python2.7/site-packages/lago/utils.py", line 58, in
> _ret_via_queue
>     queue.put({'return': func()})
>   File "/usr/lib/python2.7/site-packages/lago/prefix.py", line 1476, in
> _collect_artifacts
>     vm.collect_artifacts(path, ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 624, in
> collect_artifacts
>     ignore_nopath=ignore_nopath
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 381, in
> extract_paths
>     return self.provider.extract_paths(paths, *args, **kwargs)
>   File "/usr/lib/python2.7/site-packages/lago/providers/libvirt/vm.py", line
> 297, in extract_paths
>     ignore_nopath=ignore_nopath,
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 247, in
> extract_paths
>     self._extract_paths_scp(paths=paths, ignore_nopath=ignore_nopath)
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 266, in
> _extract_paths_scp
>     propagate_fail=False
>   File "/usr/lib/python2.7/site-packages/lago/plugins/vm.py", line 425, in
> copy_from
>     local_path=local_path,
>   File "/usr/lib/python2.7/site-packages/scp.py", line 125, in get
>     self._recv_all()
>   File "/usr/lib/python2.7/site-packages/scp.py", line 250, in _recv_all
>     msg = self.channel.recv(1024)
>   File "/usr/lib/python2.7/site-packages/paramiko/channel.py", line 615, in
> recv
>     raise socket.timeout()
> timeout
>
>
> Link to build:
> http://jenkins.ovirt.org/job/test-repo_ovirt_experimental_master/7459
>
> Thanks
> Gil