Commit 42d7c71a authored by Holger Levsen's avatar Holger Levsen

improve notification about zombie jobs

Signed-off-by: Holger Levsen's avatarHolger Levsen <holger@layer-acht.org>
parent 7fc9de6c
......@@ -164,41 +164,6 @@ if ! systemctl is-system-running > /dev/null; then
DIRTY=true
fi
#
# checks only for the main node
#
if [ "$HOSTNAME" = "$MAINNODE" ] ; then
#
# sometimes deleted jobs come back as zombies
# and we dont know why and when that happens,
# so just report those zombies here.
#
ZOMBIES=$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|odc2a|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch' || true)
if [ ! -z "$ZOMBIES" ] ; then
echo "Warning, rise of the jenkins job zombies has started again, these jobs should not exist:"
for z in $ZOMBIES ; do
echo $(basename $z)
done
DIRTY=true
echo
fi
#
# /var/log/jenkins/jenkins.log sometimes grows very fast
# and we don't yet know why, so let's monitor this for now.
JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +42G)"
if [ -z "JENKINSLOG" ] ; then
echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now."
exit 1
else
JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +23G)"
if [ -z "JENKINSLOG" ] ; then
echo "Warning, jenkins.log is larger than 23G, please do something…"
DIRTY=true
fi
fi
fi
#
# finally
#
......
......@@ -785,6 +785,42 @@ fi
sudo touch $STAMP # so on the next run, only configs newer than this file will be updated
explain "$(date) - finished deployment."
#
# some final checks only for the jenkins
#
if [ "$HOSTNAME" = "jenkins" ] ; then
#
# sometimes deleted jobs come back as zombies
# and we dont know why and when that happens,
# so just report those zombies here.
#
ZOMBIES=$(ls -1d /var/lib/jenkins/jobs/* | egrep 'strip-nondeterminism|reprotest|reproducible_(builder_(amd64|i386|armhf|arm64)|setup_(pbuilder|schroot)_testing)|chroot-installation_wheezy|aptdpkg|odc2a|stretch_install_education-thin-client-server|jessie_multiarch_versionskew|dpkg_stretch_find_trigger_cycles|sid_install_education-services|buster_install_education-services|lvc|chroot-installation_stretch_.*_upgrade_to_sid|piuparts_.*_jessie|udd_stretch' || true)
if [ ! -z "$ZOMBIES" ] ; then
figlet 'zombies!!!'
echo "Warning, rise of the jenkins job zombies has started again, these jobs should not exist:"
for z in $ZOMBIES ; do
echo $(basename $z)
done
echo
fi
#
# /var/log/jenkins/jenkins.log sometimes grows very fast
# and we don't yet know why, so let's monitor this for now.
JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +42G)"
if [ -z "JENKINSLOG" ] ; then
echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now."
exit 1
else
JENKINSLOG="$(find /var/log/jenkins -name jenkins.log -size +23G)"
if [ -z "JENKINSLOG" ] ; then
figlet 'jenkins.log size'
echo "Warning, jenkins.log is larger than 23G, please do something…"
fi
fi
fi
# finally!
case $HOSTNAME in
# set time back to the future
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment