Build failed in Jenkins: ovirt_4.1_system-tests #1
by jenkins＠jenkins.phx.ovirt.org
See <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/1/>
------------------------------------------
[...truncated 74 lines...]
Emptying some common logs
/var/log/wtmp
Done
/home/jenkins/workspace/basic_suite_master.sh-el7
/home/jenkins/workspace/test-repo_ovirt_experimental_master
Making sure there are no device mappings...
Removing the used loop devices...
Redirecting to /bin/systemctl restart libvirtd.service
---------------------------------------------------------------
Filesystem Size Used Avail Use% Mounted on
devtmpfs 24G 0 24G 0% /dev
tmpfs 24G 0 24G 0% /dev/shm
tmpfs 24G 3.2M 24G 1% /run
tmpfs 24G 0 24G 0% /sys/fs/cgroup
/dev/sda3 908G 22G 886G 3% /
tmpfs 24G 336K 24G 1% /tmp
/dev/sda1 253M 121M 132M 48% /boot
tmpfs 4.8G 0 4.8G 0% /run/user/1012
###################################################################
# Slave cleanup done #
###################################################################
[ovirt_4.1_system-tests] $ /bin/bash -xe /tmp/hudson3221576944294627045.sh
+ echo shell-scripts/system_tests.sh
shell-scripts/system_tests.sh
+ VERSION=4.1
+ SUITE_TYPE=basic
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/>
+ OVIRT_SUITE=basic_suite_4.1
+ PREFIX=<http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/ovirt-system-tests...>
+ chmod g+x <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/>
+ rm -rf <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/ovirt-system-tests...>
+ cd ovirt-system-tests
+ ../jenkins/mock_configs/mock_runner.sh --mock-confs-dir ../jenkins/mock_configs --try-proxy --execute-script automation/basic_suite_4.1.sh el7
##########################################################
##########################################################
## Mon Dec 19 14:42:52 UTC 2016 Running env: el7:epel-7-x86_64
##########################################################
ERROR: Script automation/basic_suite_4.1.sh does not exist or is not readable.
##########################################################
## Mon Dec 19 14:42:52 UTC 2016 Finished env: el7:epel-7-x86_64
## took 0 seconds
## rc = 1
##########################################################
No log files found, check command output
##!########################################################
##########################################################
Build step 'Execute shell' marked build as failure
Performing Post build task...
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -xe
echo 'shell_scripts/system_tests.collect_logs.sh'
#
# Required jjb vars:
# version
#
VERSION=4.1
SUITE_TYPE=
WORKSPACE="$PWD"
OVIRT_SUITE="$SUITE_TYPE_suite_$VERSION"
TESTS_LOGS="$WORKSPACE/ovirt-system-tests/exported-artifacts"
rm -rf "$WORKSPACE/exported-artifacts"
mkdir -p "$WORKSPACE/exported-artifacts"
if [[ -d "$TESTS_LOGS" ]]; then
mv "$TESTS_LOGS/"* "$WORKSPACE/exported-artifacts/"
fi
[ovirt_4.1_system-tests] $ /bin/bash -xe /tmp/hudson4079159423720154558.sh
+ echo shell_scripts/system_tests.collect_logs.sh
shell_scripts/system_tests.collect_logs.sh
+ VERSION=4.1
+ SUITE_TYPE=
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/>
+ OVIRT_SUITE=4.1
+ TESTS_LOGS=<http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/ovirt-system-tests...>
+ rm -rf <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/1/artifact/exported-a...>
+ mkdir -p <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/1/artifact/exported-a...>
+ [[ -d <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/ovirt-system-tests...> ]]
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -x
echo "shell-scripts/mock_cleanup.sh"
# Make clear this is the cleanup, helps reading the jenkins logs
cat <<EOC
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
EOC
shopt -s nullglob
WORKSPACE="${WORKSPACE:-$PWD}"
UMOUNT_RETRIES="${UMOUNT_RETRIES:-3}"
UMOUNT_RETRY_DELAY="${UMOUNT_RETRY_DELAY:-1s}"
safe_umount() {
local mount="${1:?}"
local attempt
for ((attempt=0 ; attempt < $UMOUNT_RETRIES ; attempt++)); do
# If this is not the 1st time through the loop, Sleep a while to let
# the problem "solve itself"
[[ attempt > 0 ]] && sleep "$UMOUNT_RETRY_DELAY"
# Try to umount
sudo umount --lazy "$mount" && return 0
# See if the mount is already not there despite failing
findmnt --kernel --first "$mount" > /dev/null && return 0
done
echo "ERROR: Failed to umount $mount."
return 1
}
# restore the permissions in the working dir, as sometimes it leaves files
# owned by root and then the 'cleanup workspace' from jenkins job fails to
# clean and breaks the jobs
sudo chown -R "$USER" "$WORKSPACE"
# Archive the logs, we want them anyway
logs=(
./*log
./*/logs
)
if [[ "$logs" ]]; then
for log in "${logs[@]}"; do
[[ "$log" = ./exported-artifacts/* ]] && continue
echo "Copying ${log} to exported-artifacts"
mv $log exported-artifacts/
done
fi
# stop any processes running inside the chroot
failed=false
mock_confs=("$WORKSPACE"/*/mocker*)
# Clean current jobs mockroot if any
for mock_conf_file in "${mock_confs[@]}"; do
[[ "$mock_conf_file" ]] || continue
echo "Cleaning up mock $mock_conf"
mock_root="${mock_conf_file##*/}"
mock_root="${mock_root%.*}"
my_mock="/usr/bin/mock"
my_mock+=" --configdir=${mock_conf_file%/*}"
my_mock+=" --root=${mock_root}"
my_mock+=" --resultdir=$WORKSPACE"
#TODO: investigate why mock --clean fails to umount certain dirs sometimes,
#so we can use it instead of manually doing all this.
echo "Killing all mock orphan processes, if any."
$my_mock \
--orphanskill \
|| {
echo "ERROR: Failed to kill orphans on $chroot."
failed=true
}
mock_root="$(\
grep \
-Po "(?<=config_opts\['root'\] = ')[^']*" \
"$mock_conf_file" \
)" || :
[[ "$mock_root" ]] || continue
mounts=($(mount | awk '{print $3}' | grep "$mock_root")) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $chroot. Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" || failed=true
done
done
# Clean any leftover chroot from other jobs
for mock_root in /var/lib/mock/*; do
this_chroot_failed=false
mounts=($(cut -d\ -f2 /proc/mounts | grep "$mock_root" | sort -r)) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $mock_root." \
"Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" && continue
# If we got here, we failed $UMOUNT_RETRIES attempts so we should make
# noise
failed=true
this_chroot_failed=true
done
if ! $this_chroot_failed; then
sudo rm -rf "$mock_root"
fi
done
# remove mock caches that are older then 2 days:
find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0 | \
xargs -0 -tr sudo rm -rf
# We make no effort to leave around caches that may still be in use because
# packages installed in them may go out of date, so may as well recreate them
# Drop all left over libvirt domains
for UUID in $(virsh list --all --uuid); do
virsh destroy $UUID || :
sleep 2
virsh undefine --remove-all-storage --storage vda --snapshots-metadata $UUID || :
done
if $failed; then
echo "Cleanup script failed, propegating failure to job"
exit 1
fi
[ovirt_4.1_system-tests] $ /bin/bash -x /tmp/hudson5498140227125921290.sh
+ echo shell-scripts/mock_cleanup.sh
shell-scripts/mock_cleanup.sh
+ cat
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
+ shopt -s nullglob
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/>
+ UMOUNT_RETRIES=3
+ UMOUNT_RETRY_DELAY=1s
+ sudo chown -R jenkins <http://jenkins.ovirt.org/job/ovirt_4.1_system-tests/ws/>
+ logs=(./*log ./*/logs)
+ [[ -n ./ovirt-system-tests/logs ]]
+ for log in '"${logs[@]}"'
+ [[ ./ovirt-system-tests/logs = ./exported-artifacts/* ]]
+ echo 'Copying ./ovirt-system-tests/logs to exported-artifacts'
Copying ./ovirt-system-tests/logs to exported-artifacts
+ mv ./ovirt-system-tests/logs exported-artifacts/
+ failed=false
+ mock_confs=("$WORKSPACE"/*/mocker*)
+ find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0
+ xargs -0 -tr sudo rm -rf
++ virsh list --all --uuid
+ false
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 1
Recording test results
ERROR: Step ?Publish JUnit test result report? failed: No test report files were found. Configuration error?
Archiving artifacts
7 years, 11 months
Build failed in Jenkins: ovirt_master_system-tests #871
by jenkins＠jenkins.phx.ovirt.org
See <http://jenkins.ovirt.org/job/ovirt_master_system-tests/871/changes>
Changes:
[Yaniv Kaul] Move log-collector to use /dev/shm for faster execution
[Evgheni Dereveanchin] OVIRT-645 remove repoproxy for Fedora and EL6 x86_64
[Barak Korren] Make mock_cleanup.sh safer about un-mouting
[Barak Korren] Fix mock_genconfig nested dict settings
------------------------------------------
[...truncated 779 lines...]
+ [[ -e <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> ]]
+ echo '----------- Cleaning with lago'
----------- Cleaning with lago
+ lago --workdir <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> destroy --yes --all-prefixes
+ echo '----------- Cleaning with lago done'
----------- Cleaning with lago done
+ [[ 0 != \0 ]]
+ echo '======== Cleanup done'
======== Cleanup done
+ exit 0
+ exit
Took 640 seconds
===================================
##!
##! ERROR ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
##!########################################################
##########################################################
Build step 'Execute shell' marked build as failure
Performing Post build task...
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -xe
echo 'shell_scripts/system_tests.collect_logs.sh'
#
# Required jjb vars:
# version
#
VERSION=master
SUITE_TYPE=
WORKSPACE="$PWD"
OVIRT_SUITE="$SUITE_TYPE_suite_$VERSION"
TESTS_LOGS="$WORKSPACE/ovirt-system-tests/exported-artifacts"
rm -rf "$WORKSPACE/exported-artifacts"
mkdir -p "$WORKSPACE/exported-artifacts"
if [[ -d "$TESTS_LOGS" ]]; then
mv "$TESTS_LOGS/"* "$WORKSPACE/exported-artifacts/"
fi
[ovirt_master_system-tests] $ /bin/bash -xe /tmp/hudson2709353388353252686.sh
+ echo shell_scripts/system_tests.collect_logs.sh
shell_scripts/system_tests.collect_logs.sh
+ VERSION=master
+ SUITE_TYPE=
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/>
+ OVIRT_SUITE=master
+ TESTS_LOGS=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...>
+ rm -rf <http://jenkins.ovirt.org/job/ovirt_master_system-tests/871/artifact/expor...>
+ mkdir -p <http://jenkins.ovirt.org/job/ovirt_master_system-tests/871/artifact/expor...>
+ [[ -d <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> ]]
+ mv <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> <http://jenkins.ovirt.org/job/ovirt_master_system-tests/871/artifact/expor...>
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -x
echo "shell-scripts/mock_cleanup.sh"
# Make clear this is the cleanup, helps reading the jenkins logs
cat <<EOC
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
EOC
shopt -s nullglob
WORKSPACE="${WORKSPACE:-$PWD}"
UMOUNT_RETRIES="${UMOUNT_RETRIES:-3}"
UMOUNT_RETRY_DELAY="${UMOUNT_RETRY_DELAY:-1s}"
safe_umount() {
local mount="${1:?}"
local attempt
for ((attempt=0 ; attempt < $UMOUNT_RETRIES ; attempt++)); do
# If this is not the 1st time through the loop, Sleep a while to let
# the problem "solve itself"
[[ attempt > 0 ]] && sleep "$UMOUNT_RETRY_DELAY"
# Try to umount
sudo umount --lazy "$mount" && return 0
# See if the mount is already not there despite failing
findmnt --kernel --first "$mount" > /dev/null && return 0
done
echo "ERROR: Failed to umount $mount."
return 1
}
# restore the permissions in the working dir, as sometimes it leaves files
# owned by root and then the 'cleanup workspace' from jenkins job fails to
# clean and breaks the jobs
sudo chown -R "$USER" "$WORKSPACE"
# Archive the logs, we want them anyway
logs=(
./*log
./*/logs
)
if [[ "$logs" ]]; then
for log in "${logs[@]}"; do
[[ "$log" = ./exported-artifacts/* ]] && continue
echo "Copying ${log} to exported-artifacts"
mv $log exported-artifacts/
done
fi
# stop any processes running inside the chroot
failed=false
mock_confs=("$WORKSPACE"/*/mocker*)
# Clean current jobs mockroot if any
for mock_conf_file in "${mock_confs[@]}"; do
[[ "$mock_conf_file" ]] || continue
echo "Cleaning up mock $mock_conf"
mock_root="${mock_conf_file##*/}"
mock_root="${mock_root%.*}"
my_mock="/usr/bin/mock"
my_mock+=" --configdir=${mock_conf_file%/*}"
my_mock+=" --root=${mock_root}"
my_mock+=" --resultdir=$WORKSPACE"
#TODO: investigate why mock --clean fails to umount certain dirs sometimes,
#so we can use it instead of manually doing all this.
echo "Killing all mock orphan processes, if any."
$my_mock \
--orphanskill \
|| {
echo "ERROR: Failed to kill orphans on $chroot."
failed=true
}
mock_root="$(\
grep \
-Po "(?<=config_opts\['root'\] = ')[^']*" \
"$mock_conf_file" \
)" || :
[[ "$mock_root" ]] || continue
mounts=($(mount | awk '{print $3}' | grep "$mock_root")) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $chroot. Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" || failed=true
done
done
# Clean any leftover chroot from other jobs
for mock_root in /var/lib/mock/*; do
this_chroot_failed=false
mounts=($(cut -d\ -f2 /proc/mounts | grep "$mock_root" | sort -r)) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $mock_root." \
"Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" && continue
# If we got here, we failed $UMOUNT_RETRIES attempts so we should make
# noise
failed=true
this_chroot_failed=true
done
if ! $this_chroot_failed; then
sudo rm -rf "$mock_root"
fi
done
# remove mock caches that are older then 2 days:
find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0 | \
xargs -0 -tr sudo rm -rf
# We make no effort to leave around caches that may still be in use because
# packages installed in them may go out of date, so may as well recreate them
# Drop all left over libvirt domains
for UUID in $(virsh list --all --uuid); do
virsh destroy $UUID || :
sleep 2
virsh undefine --remove-all-storage --storage vda --snapshots-metadata $UUID || :
done
if $failed; then
echo "Cleanup script failed, propegating failure to job"
exit 1
fi
[ovirt_master_system-tests] $ /bin/bash -x /tmp/hudson7578817524005175999.sh
+ echo shell-scripts/mock_cleanup.sh
shell-scripts/mock_cleanup.sh
+ cat
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
+ shopt -s nullglob
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/>
+ UMOUNT_RETRIES=3
+ UMOUNT_RETRY_DELAY=1s
+ sudo chown -R jenkins <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/>
+ logs=(./*log ./*/logs)
+ [[ -n ./ovirt-system-tests/logs ]]
+ for log in '"${logs[@]}"'
+ [[ ./ovirt-system-tests/logs = ./exported-artifacts/* ]]
+ echo 'Copying ./ovirt-system-tests/logs to exported-artifacts'
Copying ./ovirt-system-tests/logs to exported-artifacts
+ mv ./ovirt-system-tests/logs exported-artifacts/
+ failed=false
+ mock_confs=("$WORKSPACE"/*/mocker*)
+ for mock_conf_file in '"${mock_confs[@]}"'
+ [[ -n <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...> ]]
+ echo 'Cleaning up mock '
Cleaning up mock
+ mock_root=mocker-epel-7-x86_64.el7.cfg
+ mock_root=mocker-epel-7-x86_64.el7
+ my_mock=/usr/bin/mock
+ my_mock+=' --configdir=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-tests'>
+ my_mock+=' --root=mocker-epel-7-x86_64.el7'
+ my_mock+=' --resultdir=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/'>
+ echo 'Killing all mock orphan processes, if any.'
Killing all mock orphan processes, if any.
+ /usr/bin/mock --configdir=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-tests> --root=mocker-epel-7-x86_64.el7 --resultdir=<http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/> --orphanskill
WARNING: Could not find required logging config file: <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te....> Using default...
INFO: mock.py version 1.2.21 starting (python version = 3.5.1)...
Start: init plugins
INFO: selinux enabled
Finish: init plugins
Start: run
Finish: run
++ grep -Po '(?<=config_opts\['\''root'\''\] = '\'')[^'\'']*' <http://jenkins.ovirt.org/job/ovirt_master_system-tests/ws/ovirt-system-te...>
+ mock_root=epel-7-x86_64-b8fbc97503d2f9478015768d3094ffe4
+ [[ -n epel-7-x86_64-b8fbc97503d2f9478015768d3094ffe4 ]]
+ mounts=($(mount | awk '{print $3}' | grep "$mock_root"))
++ mount
++ awk '{print $3}'
++ grep epel-7-x86_64-b8fbc97503d2f9478015768d3094ffe4
+ :
+ [[ -n '' ]]
+ find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0
+ xargs -0 -tr sudo rm -rf
++ virsh list --all --uuid
+ false
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 1
Recording test results
Archiving artifacts
7 years, 11 months
Ability to run [job-name] when a tag is pushed to Gerrit
by Vojtech Szocs
Hi,
this was discussed loong time ago.. how much effort would it take
for oVirt CI infra to support running e.g. [build-artifacts] when
a git tag is pushed to Gerrit repo?
Imagine the following situation:
1, new tag is pushed -or- existing tag gets updated (force push)
2, CI infra detects the new/updated tag and runs [job-name-here]
on the relevant (tagged) commit
I'm asking this because in oVirt Dashboard, we're making releases
from tagged commits, so we basically:
- merge patch that prepares the (stable) branch for release
- tag the release-prep patch & push tag to Gerrit
- re-trigger [build-artifacts] for the release-prep patch
Thanks,
Vojtech
7 years, 11 months
oVirt 4.1.0 beta2 merge / branch / tag / bugzilla reminder
by Sandro Bonazzola
All stable branch maintainers, please make sure to merge all relevant open
bugs until Wednesday December 21st morning 11:00 AM TLV time (10:00 AM CET).
In order to stabilize the build and allow to prepare jenkins on time, 4.1
branches need to be created before or on Monday Dec 19th.
oVirt engine will be branched on Monday Dec 19th at 11:00 AM TLV time
(10:00 AM CET).
This build is also marking the string freeze milestone. After this build
please avoid to push new strings to be translated to allow translators to
complete translations for RC.
For each package that need to be built (i.e oVirt product) please make sure
every bug in MODIFIED has the right Target Release and Target Milestone.
A Target release should state the final version of the package you're
building (e.g. for ovirt-engine, tag: ovirt-engine-4.1.0_beta2, tr: 4.1.0)
A list of bugs that require attention is here:
https://bugzilla.redhat.com/buglist.cgi?quicksearch=target_milestone%3A4....
--
Sandro Bonazzola
Better technology. Faster innovation. Powered by community collaboration.
See how it works at redhat.com
7 years, 11 months
Build failed in Jenkins: ovirt_3.6_system-tests #827
by jenkins＠jenkins.phx.ovirt.org
See <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/827/changes>
Changes:
[Yaniv Kaul] Move log-collector to use /dev/shm for faster execution
[Sandro Bonazzola] repo_closure_check: fix el7 repos for metrics
[Sandro Bonazzola] repos-check-closure: add 4.1 repos
[Sandro Bonazzola] ovirt-live: add 4.1 branch jobs
------------------------------------------
[...truncated 789 lines...]
+ [[ -e <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> ]]
+ echo '----------- Cleaning with lago'
----------- Cleaning with lago
+ lago --workdir <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> destroy --yes --all-prefixes
+ echo '----------- Cleaning with lago done'
----------- Cleaning with lago done
+ [[ 0 != \0 ]]
+ echo '======== Cleanup done'
======== Cleanup done
+ exit 0
+ exit
Took 528 seconds
===================================
##!
##! ERROR ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
##!########################################################
##########################################################
Build step 'Execute shell' marked build as failure
Performing Post build task...
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -xe
echo 'shell_scripts/system_tests.collect_logs.sh'
#
# Required jjb vars:
# version
#
VERSION=3.6
SUITE_TYPE=
WORKSPACE="$PWD"
OVIRT_SUITE="$SUITE_TYPE_suite_$VERSION"
TESTS_LOGS="$WORKSPACE/ovirt-system-tests/exported-artifacts"
rm -rf "$WORKSPACE/exported-artifacts"
mkdir -p "$WORKSPACE/exported-artifacts"
if [[ -d "$TESTS_LOGS" ]]; then
mv "$TESTS_LOGS/"* "$WORKSPACE/exported-artifacts/"
fi
[ovirt_3.6_system-tests] $ /bin/bash -xe /tmp/hudson8592848801410316506.sh
+ echo shell_scripts/system_tests.collect_logs.sh
shell_scripts/system_tests.collect_logs.sh
+ VERSION=3.6
+ SUITE_TYPE=
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/>
+ OVIRT_SUITE=3.6
+ TESTS_LOGS=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...>
+ rm -rf <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/827/artifact/exported...>
+ mkdir -p <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/827/artifact/exported...>
+ [[ -d <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> ]]
+ mv <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/827/artifact/exported...>
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -x
echo "shell-scripts/mock_cleanup.sh"
# Make clear this is the cleanup, helps reading the jenkins logs
cat <<EOC
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
EOC
shopt -s nullglob
WORKSPACE="${WORKSPACE:-$PWD}"
UMOUNT_RETRIES="${UMOUNT_RETRIES:-3}"
UMOUNT_RETRY_DELAY="${UMOUNT_RETRY_DELAY:-1s}"
safe_umount() {
local mount="${1:?}"
local attempt
for ((attempt=0 ; attempt < $UMOUNT_RETRIES ; attempt++)); do
# If this is not the 1st time through the loop, Sleep a while to let
# the problem "solve itself"
[[ attempt > 0 ]] && sleep "$UMOUNT_RETRY_DELAY"
# Try to umount
sudo umount --lazy "$mount" && return 0
# See if the mount is already not there despite failing
findmnt --kernel --first "$mount" > /dev/null && return 0
done
echo "ERROR: Failed to umount $mount."
return 1
}
# restore the permissions in the working dir, as sometimes it leaves files
# owned by root and then the 'cleanup workspace' from jenkins job fails to
# clean and breaks the jobs
sudo chown -R "$USER" "$WORKSPACE"
# Archive the logs, we want them anyway
logs=(
./*log
./*/logs
)
if [[ "$logs" ]]; then
for log in "${logs[@]}"; do
[[ "$log" = ./exported-artifacts/* ]] && continue
echo "Copying ${log} to exported-artifacts"
mv $log exported-artifacts/
done
fi
# stop any processes running inside the chroot
failed=false
mock_confs=("$WORKSPACE"/*/mocker*)
# Clean current jobs mockroot if any
for mock_conf_file in "${mock_confs[@]}"; do
[[ "$mock_conf_file" ]] || continue
echo "Cleaning up mock $mock_conf"
mock_root="${mock_conf_file##*/}"
mock_root="${mock_root%.*}"
my_mock="/usr/bin/mock"
my_mock+=" --configdir=${mock_conf_file%/*}"
my_mock+=" --root=${mock_root}"
my_mock+=" --resultdir=$WORKSPACE"
#TODO: investigate why mock --clean fails to umount certain dirs sometimes,
#so we can use it instead of manually doing all this.
echo "Killing all mock orphan processes, if any."
$my_mock \
--orphanskill \
|| {
echo "ERROR: Failed to kill orphans on $chroot."
failed=true
}
mock_root="$(\
grep \
-Po "(?<=config_opts\['root'\] = ')[^']*" \
"$mock_conf_file" \
)" || :
[[ "$mock_root" ]] || continue
mounts=($(mount | awk '{print $3}' | grep "$mock_root")) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $chroot. Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" || failed=true
done
done
# Clean any leftover chroot from other jobs
for mock_root in /var/lib/mock/*; do
this_chroot_failed=false
mounts=($(cut -d\ -f2 /proc/mounts | grep "$mock_root" | sort -r)) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $mock_root." \
"Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" && continue
# If we got here, we failed $UMOUNT_RETRIES attempts so we should make
# noise
failed=true
this_chroot_failed=true
done
if ! $this_chroot_failed; then
sudo rm -rf "$mock_root"
fi
done
# remove mock caches that are older then 2 days:
find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0 | \
xargs -0 -tr sudo rm -rf
# We make no effort to leave around caches that may still be in use because
# packages installed in them may go out of date, so may as well recreate them
# Drop all left over libvirt domains
for UUID in $(virsh list --all --uuid); do
virsh destroy $UUID || :
sleep 2
virsh undefine --remove-all-storage --storage vda --snapshots-metadata $UUID || :
done
if $failed; then
echo "Cleanup script failed, propegating failure to job"
exit 1
fi
[ovirt_3.6_system-tests] $ /bin/bash -x /tmp/hudson5394810375570137830.sh
+ echo shell-scripts/mock_cleanup.sh
shell-scripts/mock_cleanup.sh
+ cat
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
+ shopt -s nullglob
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/>
+ UMOUNT_RETRIES=3
+ UMOUNT_RETRY_DELAY=1s
+ sudo chown -R jenkins <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/>
+ logs=(./*log ./*/logs)
+ [[ -n ./ovirt-system-tests/logs ]]
+ for log in '"${logs[@]}"'
+ [[ ./ovirt-system-tests/logs = ./exported-artifacts/* ]]
+ echo 'Copying ./ovirt-system-tests/logs to exported-artifacts'
Copying ./ovirt-system-tests/logs to exported-artifacts
+ mv ./ovirt-system-tests/logs exported-artifacts/
+ failed=false
+ mock_confs=("$WORKSPACE"/*/mocker*)
+ for mock_conf_file in '"${mock_confs[@]}"'
+ [[ -n <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...> ]]
+ echo 'Cleaning up mock '
Cleaning up mock
+ mock_root=mocker-epel-7-x86_64.el7.cfg
+ mock_root=mocker-epel-7-x86_64.el7
+ my_mock=/usr/bin/mock
+ my_mock+=' --configdir=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests'>
+ my_mock+=' --root=mocker-epel-7-x86_64.el7'
+ my_mock+=' --resultdir=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/'>
+ echo 'Killing all mock orphan processes, if any.'
Killing all mock orphan processes, if any.
+ /usr/bin/mock --configdir=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests> --root=mocker-epel-7-x86_64.el7 --resultdir=<http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/> --orphanskill
WARNING: Could not find required logging config file: <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests....> Using default...
INFO: mock.py version 1.2.21 starting (python version = 3.5.2)...
Start: init plugins
INFO: selinux enabled
Finish: init plugins
Start: run
Finish: run
++ grep -Po '(?<=config_opts\['\''root'\''\] = '\'')[^'\'']*' <http://jenkins.ovirt.org/job/ovirt_3.6_system-tests/ws/ovirt-system-tests...>
+ mock_root=epel-7-x86_64-aa7d25407b9c15b3736f25fc4fcf3710
+ [[ -n epel-7-x86_64-aa7d25407b9c15b3736f25fc4fcf3710 ]]
+ mounts=($(mount | awk '{print $3}' | grep "$mock_root"))
++ mount
++ awk '{print $3}'
++ grep epel-7-x86_64-aa7d25407b9c15b3736f25fc4fcf3710
+ :
+ [[ -n '' ]]
+ find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0
+ xargs -0 -tr sudo rm -rf
++ virsh list --all --uuid
+ false
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 1
Recording test results
Archiving artifacts
7 years, 11 months
Build failed in Jenkins: ovirt_4.0_system-tests #746
by jenkins＠jenkins.phx.ovirt.org
See <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/746/changes>
Changes:
[Sandro Bonazzola] 3.6 repo: Fix typo: ovirt-optimizier
------------------------------------------
[...truncated 765 lines...]
+ [[ -e <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> ]]
+ echo '----------- Cleaning with lago'
----------- Cleaning with lago
+ lago --workdir <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> destroy --yes --all-prefixes
+ echo '----------- Cleaning with lago done'
----------- Cleaning with lago done
+ [[ 0 != \0 ]]
+ echo '======== Cleanup done'
======== Cleanup done
+ exit 0
+ exit
Took 560 seconds
===================================
##!
##! ERROR ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
##!########################################################
##########################################################
Build step 'Execute shell' marked build as failure
Performing Post build task...
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -xe
echo 'shell_scripts/system_tests.collect_logs.sh'
#
# Required jjb vars:
# version
#
VERSION=4.0
SUITE_TYPE=
WORKSPACE="$PWD"
OVIRT_SUITE="$SUITE_TYPE_suite_$VERSION"
TESTS_LOGS="$WORKSPACE/ovirt-system-tests/exported-artifacts"
rm -rf "$WORKSPACE/exported-artifacts"
mkdir -p "$WORKSPACE/exported-artifacts"
if [[ -d "$TESTS_LOGS" ]]; then
mv "$TESTS_LOGS/"* "$WORKSPACE/exported-artifacts/"
fi
[ovirt_4.0_system-tests] $ /bin/bash -xe /tmp/hudson5562949287383041800.sh
+ echo shell_scripts/system_tests.collect_logs.sh
shell_scripts/system_tests.collect_logs.sh
+ VERSION=4.0
+ SUITE_TYPE=
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/>
+ OVIRT_SUITE=4.0
+ TESTS_LOGS=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...>
+ rm -rf <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/746/artifact/exported...>
+ mkdir -p <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/746/artifact/exported...>
+ [[ -d <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> ]]
+ mv <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/746/artifact/exported...>
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 0
Match found for :.* : True
Logical operation result is TRUE
Running script : #!/bin/bash -x
echo "shell-scripts/mock_cleanup.sh"
# Make clear this is the cleanup, helps reading the jenkins logs
cat <<EOC
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
EOC
shopt -s nullglob
WORKSPACE="${WORKSPACE:-$PWD}"
UMOUNT_RETRIES="${UMOUNT_RETRIES:-3}"
UMOUNT_RETRY_DELAY="${UMOUNT_RETRY_DELAY:-1s}"
safe_umount() {
local mount="${1:?}"
local attempt
for ((attempt=0 ; attempt < $UMOUNT_RETRIES ; attempt++)); do
# If this is not the 1st time through the loop, Sleep a while to let
# the problem "solve itself"
[[ attempt > 0 ]] && sleep "$UMOUNT_RETRY_DELAY"
# Try to umount
sudo umount --lazy "$mount" && return 0
# See if the mount is already not there despite failing
findmnt --kernel --first "$mount" > /dev/null && return 0
done
echo "ERROR: Failed to umount $mount."
return 1
}
# restore the permissions in the working dir, as sometimes it leaves files
# owned by root and then the 'cleanup workspace' from jenkins job fails to
# clean and breaks the jobs
sudo chown -R "$USER" "$WORKSPACE"
# Archive the logs, we want them anyway
logs=(
./*log
./*/logs
)
if [[ "$logs" ]]; then
for log in "${logs[@]}"; do
[[ "$log" = ./exported-artifacts/* ]] && continue
echo "Copying ${log} to exported-artifacts"
mv $log exported-artifacts/
done
fi
# stop any processes running inside the chroot
failed=false
mock_confs=("$WORKSPACE"/*/mocker*)
# Clean current jobs mockroot if any
for mock_conf_file in "${mock_confs[@]}"; do
[[ "$mock_conf_file" ]] || continue
echo "Cleaning up mock $mock_conf"
mock_root="${mock_conf_file##*/}"
mock_root="${mock_root%.*}"
my_mock="/usr/bin/mock"
my_mock+=" --configdir=${mock_conf_file%/*}"
my_mock+=" --root=${mock_root}"
my_mock+=" --resultdir=$WORKSPACE"
#TODO: investigate why mock --clean fails to umount certain dirs sometimes,
#so we can use it instead of manually doing all this.
echo "Killing all mock orphan processes, if any."
$my_mock \
--orphanskill \
|| {
echo "ERROR: Failed to kill orphans on $chroot."
failed=true
}
mock_root="$(\
grep \
-Po "(?<=config_opts\['root'\] = ')[^']*" \
"$mock_conf_file" \
)" || :
[[ "$mock_root" ]] || continue
mounts=($(mount | awk '{print $3}' | grep "$mock_root")) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $chroot. Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" || failed=true
done
done
# Clean any leftover chroot from other jobs
for mock_root in /var/lib/mock/*; do
this_chroot_failed=false
mounts=($(cut -d\ -f2 /proc/mounts | grep "$mock_root" | sort -r)) || :
if [[ "$mounts" ]]; then
echo "Found mounted dirs inside the chroot $mock_root." \
"Trying to umount."
fi
for mount in "${mounts[@]}"; do
safe_umount "$mount" && continue
# If we got here, we failed $UMOUNT_RETRIES attempts so we should make
# noise
failed=true
this_chroot_failed=true
done
if ! $this_chroot_failed; then
sudo rm -rf "$mock_root"
fi
done
# remove mock caches that are older then 2 days:
find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0 | \
xargs -0 -tr sudo rm -rf
# We make no effort to leave around caches that may still be in use because
# packages installed in them may go out of date, so may as well recreate them
# Drop all left over libvirt domains
for UUID in $(virsh list --all --uuid); do
virsh destroy $UUID || :
sleep 2
virsh undefine --remove-all-storage --storage vda --snapshots-metadata $UUID || :
done
if $failed; then
echo "Cleanup script failed, propegating failure to job"
exit 1
fi
[ovirt_4.0_system-tests] $ /bin/bash -x /tmp/hudson5757423838546789539.sh
+ echo shell-scripts/mock_cleanup.sh
shell-scripts/mock_cleanup.sh
+ cat
_______________________________________________________________________
#######################################################################
# #
# CLEANUP #
# #
#######################################################################
+ shopt -s nullglob
+ WORKSPACE=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/>
+ UMOUNT_RETRIES=3
+ UMOUNT_RETRY_DELAY=1s
+ sudo chown -R jenkins <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/>
+ logs=(./*log ./*/logs)
+ [[ -n ./ovirt-system-tests/logs ]]
+ for log in '"${logs[@]}"'
+ [[ ./ovirt-system-tests/logs = ./exported-artifacts/* ]]
+ echo 'Copying ./ovirt-system-tests/logs to exported-artifacts'
Copying ./ovirt-system-tests/logs to exported-artifacts
+ mv ./ovirt-system-tests/logs exported-artifacts/
+ failed=false
+ mock_confs=("$WORKSPACE"/*/mocker*)
+ for mock_conf_file in '"${mock_confs[@]}"'
+ [[ -n <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...> ]]
+ echo 'Cleaning up mock '
Cleaning up mock
+ mock_root=mocker-epel-7-x86_64.el7.cfg
+ mock_root=mocker-epel-7-x86_64.el7
+ my_mock=/usr/bin/mock
+ my_mock+=' --configdir=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests'>
+ my_mock+=' --root=mocker-epel-7-x86_64.el7'
+ my_mock+=' --resultdir=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/'>
+ echo 'Killing all mock orphan processes, if any.'
Killing all mock orphan processes, if any.
+ /usr/bin/mock --configdir=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests> --root=mocker-epel-7-x86_64.el7 --resultdir=<http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/> --orphanskill
WARNING: Could not find required logging config file: <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests....> Using default...
INFO: mock.py version 1.2.21 starting (python version = 3.5.1)...
Start: init plugins
INFO: selinux enabled
Finish: init plugins
Start: run
Finish: run
++ grep -Po '(?<=config_opts\['\''root'\''\] = '\'')[^'\'']*' <http://jenkins.ovirt.org/job/ovirt_4.0_system-tests/ws/ovirt-system-tests...>
+ mock_root=epel-7-x86_64-ed452d1ea0f6dadf50c66d3b3f6b3a6e
+ [[ -n epel-7-x86_64-ed452d1ea0f6dadf50c66d3b3f6b3a6e ]]
+ mounts=($(mount | awk '{print $3}' | grep "$mock_root"))
++ mount
++ awk '{print $3}'
++ grep epel-7-x86_64-ed452d1ea0f6dadf50c66d3b3f6b3a6e
+ :
+ [[ -n '' ]]
+ find /var/cache/mock/ -mindepth 1 -maxdepth 1 -type d -mtime +2 -print0
+ xargs -0 -tr sudo rm -rf
++ virsh list --all --uuid
+ false
POST BUILD TASK : SUCCESS
END OF POST BUILD TASK : 1
Recording test results
Archiving artifacts
7 years, 11 months
