djm: "documented jenkins maintenance", script to support logged node maintenance

Signed-off-by: Holger Levsen <holger@layer-acht.org>

bin/djm

+#!/bin/bash
+# vim: set noexpandtab:
+
+#
+# djm - documented jenkins maintenance
+#
+# Copyright 2023 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv2
+#
+
+set -e
+set -o pipefail		# see eg http://petereisentraut.blogspot.com/2010/11/pipefail.html
+
+#
+# define environment and parse parameters
+#
+
+LOGFILE=~/.djm.log
+TARGET=$1
+ACTION=$2
+REASON=$3
+DRY_RUN=$4
+FAIL_REASON=""
+
+if [ -z "$TARGET" ] ; then
+	FAIL_REASON="${FAIL_REASON}Undefined target.\n"
+else case $TARGET in
+	all)	TARGET="$(./nodes/list_nodes )"
+		;;
+	*)	TARGET="$(./nodes/list_nodes | egrep $TARGET || true)"
+		;;
+	esac
+fi
+
+if [ -z "$TARGET" ] ; then
+	FAIL_REASON="${FAIL_REASON}Undefined target.\n"
+fi
+
+case $ACTION in
+	r|reboot)	ACTION=reboot ;;
+	p|powercycle)	ACTION=powercycle ;;
+	a|autoremove)	ACTION=autoremove ;;
+	b|bash)		ACTION=bash ;;
+	*)		FAIL_REASON="${FAIL_REASON}Undefined action.\n"
+			;;
+esac
+
+case $REASON in
+	ku)	REASON="kernel upgrade" ;;
+	np)	REASON="no ping" ;;
+	rm)	REASON="regular maintenance" ;;
+	*)		FAIL_REASON="${FAIL_REASON}Undefined reason.\n"
+			;;
+esac
+
+if [ -z "$DRY_RUN" ] ; then
+	DRY_MODE=false
+else
+	DRY_MODE=true
+fi
+
+#
+# display help text in case of problems (or no params given)
+#
+
+if [ -n "$FAIL_REASON" ] ; then
+	echo
+	echo problem parsing parameters, djm needs at least three:
+	echo
+	echo "TARGET=\$1	'all' or grepable (jenkins, amd64, ionos, osuosl3) from ./nodes/list_nodes"
+	echo "ACTION=\$2	r_eboot, _p_owercycle, _a_utoremove, _b_bash (for manual maintenance)"
+	echo "REASON=\$3	ku=kernel update, np=no pings, rm=regular maintenance - TODO: define more"
+	echo "DRY_RUN=\$4	optional to force dry-run mode"
+	echo
+	echo params given:
+	echo
+	echo "\$1 = $1"
+	echo "\$2 = $2"
+	echo "\$3 = $3"
+	echo "\$4 = $4"
+	echo
+	echo parsing problems:
+	echo
+	echo -e "$FAIL_REASON"
+	echo
+	exit 1
+fi
+
+#
+# main
+#
+
+if $DRY_MODE ; then
+	echo "###"
+	echo "###"
+	echo "###  running in dry-run mode, not doing anything for real."
+	echo "###"
+	echo "###"
+else
+	echo
+	echo Please confirm you want to do this:
+	echo
+fi
+
+for NODE in $TARGET ; do
+	SHORTNODE=$(echo $NODE | cut -d '.' -f1)
+	echo $(date), $SHORTNODE, $ACTION, $REASON
+done
+
+if $DRY_MODE ; then
+	echo
+	echo "# dry-run end."
+	exit 0
+else
+	echo
+	echo -n "confirm: y/N: "
+	read CONFIRM
+	echo
+	case $CONFIRM in
+		y|Y)	: ;;
+		*)	echo "Ok, aborting" ; exit 0 ;;
+	esac
+fi
+
+BG=""
+get_arch_color() {
+	case "$1" in
+		*amd64*)		BG=lightgreen ;;
+		*i386*)			BG=lightblue ;;
+		*arm64*)		BG=orange ;;
+		*armhf*)		BG=lightyellow ;;
+		*jenkins.debian.*)	BG=yellow ;;
+		*)			BG=white ;;
+	esac
+}
+
+LOGDATE=$(date)
+for NODE in $TARGET ; do
+	SHORTNODE=$(echo $NODE | cut -d '.' -f1)
+	echo $(date), $SHORTNODE, $ACTION, $REASON
+	get_arch_color $NODE
+	#
+	# action
+	#
+	case $ACTION in
+		reboot)	ssh $NODE "sudo reboot" || true
+			TMPFILE=$(mktemp --tmpdir=/tmp djm-XXXXXXXX)
+			cat > $TMPFILE << EOF
+#!/bin/bash
+for i in $(seq 1 42 | xargs echo) ; do
+	echo -n .
+	sleep 1
+done
+while ! ssh $NODE 'uptime ; echo press enter ; read a' ; do 
+	for i in $(seq 1 30 | xargs echo) ; do 
+		echo -n "."
+		sleep 1
+	done
+done
+rm $TMPFILE
+EOF
+			echo xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $TMPFILE" 
+			xterm -T "$SHORTNODE / reboot ($TMPFILE)" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "bash $TMPFILE" &
+			echo
+			;;
+		powercycle)	echo FIXME, not yet implemented.
+				exit 1
+				;;
+		autoremove)	xterm -T "$SHORTNODE / autoremove" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE 'sudo apt-get autoremove ; sudo apt-get clean'" &
+			;;
+		bash)	xterm -T "$SHORTNODE / bash" -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh $NODE" &
+			;;
+		*)	echo unsupported action, sorry, please improve.
+			exit 1
+			;;
+	esac
+
+	#
+	# log
+	#
+	echo "$LOGDATE, $SHORTNODE, $ACTION, $REASON" >> $LOGFILE
+done
+
+echo the end