more stabbing after re-reading https://jenkins.io/blog/2016/11/21/gc-tuning/ (45894bb4) · Commits · Debian QA / jenkins.debian.net

bin/reproducible_common.sh

+98 −0

Original line number	Diff line number	Diff line
		@@ -857,6 +857,104 @@ create_debian_png_from_table() {
		rm ${TABLE[$1]}.csv
		}

		#
		# create the png (and query the db to populate a csv file...) for Arch Linux
		#
		create_archlinux_png_from_table() {
		echo "Checking whether to update $2..."
		# $1 = id of the stats table
		# $2 = image file name
		echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
		# prepare query
		WHERE_EXTRA="WHERE suite = '$SUITE'"
		if [ $1 -eq 0 ] \|\| [ $1 -eq 2 ] ; then
		# TABLE[0+2] have a architecture column:
		WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
		fi
		# run query
		if [ $1 -eq 1 ] ; then
		# not sure if it's worth to generate the following query...
		WHERE_EXTRA="AND architecture='$ARCH'"

		# This query became much more obnoxious when gaining
		# compatibility with postgres
		query_to_csv "SELECT stats.datum,
		COALESCE(reproducible_stretch,0) AS reproducible_stretch,
		COALESCE(reproducible_buster,0) AS reproducible_buster,
		COALESCE(reproducible_unstable,0) AS reproducible_unstable,
		COALESCE(reproducible_experimental,0) AS reproducible_experimental,
		COALESCE(FTBR_stretch,0) AS FTBR_stretch,
		COALESCE(FTBR_buster,0) AS FTBR_buster,
		COALESCE(FTBR_unstable,0) AS FTBR_unstable,
		COALESCE(FTBR_experimental,0) AS FTBR_experimental,
		COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
		COALESCE(FTBFS_buster,0) AS FTBFS_buster,
		COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
		COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
		COALESCE(other_stretch,0) AS other_stretch,
		COALESCE(other_buster,0) AS other_buster,
		COALESCE(other_unstable,0) AS other_unstable,
		COALESCE(other_experimental,0) AS other_experimental
		FROM (SELECT s.datum,
		COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
		COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
		COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
		COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
		(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
		(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
		(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
		(SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
		(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
		(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
		(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
		(SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
		(SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
		(SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
		(SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
		(SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
		FROM stats_builds_per_day AS s GROUP BY s.datum) as stats
		ORDER BY datum" >> ${TABLE[$1]}.csv
		elif [ $1 -eq 2 ] ; then
		# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
		query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
		else
		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
		fi
		# this is a gross hack: normally we take the number of colors a table should have...
		# for the builds_age table we only want one color, but different ones, so this hack:
		COLORS=${COLOR[$1]}
		if [ $1 -eq 2 ] ; then
		case "$SUITE" in
		stretch) COLORS=40 ;;
		buster) COLORS=41 ;;
		unstable) COLORS=42 ;;
		experimental) COLORS=43 ;;
		esac
		fi
		local WIDTH=1920
		local HEIGHT=960
		# only generate graph if the query returned data
		if [ $(cat ${TABLE[$1]}.csv \| wc -l) -gt 1 ] ; then
		echo "Updating $2..."
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Generating $2."
		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
		mv $2 $ARCHBASE/$DIR
		[ "$DIR" = "." ] \|\| rmdir $(dirname $2)
		# create empty dummy png if there havent been any results ever
		elif [ ! -f $ARCHBASE/$DIR/$(basename $2) ] ; then
		DIR=$(dirname $2)
		mkdir -p $DIR
		echo "Creating $2 dummy."
		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
		mv $2 $ARCHBASE/$DIR
		[ "$DIR" = "." ] \|\| rmdir $(dirname $2)
		fi
		rm ${TABLE[$1]}.csv
		}


		find_in_buildlogs() {
		egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
		}

bin/reproducible_html_archlinux.sh

+9 −0

Original line number	Diff line number	Diff line
		@@ -437,6 +437,15 @@ if [ -z "$1" ] ; then
		ARCHLINUX_NR_UNKNOWN=0
		WIDTH=1920
		HEIGHT=960
		# variables related to the stats we update
		# FIELDS[0]="datum, reproducible, FTBR, FTBFS, other, untested" # FIXME: for this Arch Linux still uses a .csv file...
		FIELDS[1]="datum"
		for i in reproducible FTBR FTBFS other ; do
		for j in $SUITES ; do
		FIELDS[1]="${FIELDS[1]}, ${i}_${j}"
		done
		done
		FIELDS[2]="datum, oldest"

		repostats
		dashboard_page

hosts/jenkins/etc/default/jenkins

+3 −3

Original line number	Diff line number	Diff line
		@@ -17,7 +17,7 @@ JAVA_ARGS="-Djava.awt.headless=true"
		#JAVA_ARGS="-Xmx256m"
		#JAVA_ARGS="-Djava.net.preferIPv4Stack=true" # make jenkins listen on IPv4 address
		# See some recommendation from https://jenkins.io/blog/2016/11/21/gc-tuning/
		JAVA_ARGS="$JAVA_ARGS -Xms4G -Xmx8G" # heap settings
		JAVA_ARGS="$JAVA_ARGS -Xms8G -Xmx24G" # heap settings
		JAVA_ARGS="$JAVA_ARGS -server -XX:+AlwaysPreTouch"
		# store heap dumps in case of OOM, to investigate it later
		JAVA_ARGS="$JAVA_ARGS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$JENKINS_HOME/heapDumps/"
		@@ -30,8 +30,8 @@ JAVA_ARGS="$JAVA_ARGS -XX:+PrintGCCause -XX:+PrintTenuringDistribution -XX:+Prin
		JAVA_ARGS="$JAVA_ARGS -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication"
		JAVA_ARGS="$JAVA_ARGS -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1"
		# h01ger trying stuff
		JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops"
		MALLOC_ARENA_MAX=2
		JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops -XX:+UseCompressedClassPointers"
		export MALLOC_ARENA_MAX=2

		# make jenkins listen on IPv4 address
		#JAVA_ARGS="-Djava.net.preferIPv4Stack=true"