more stabbing after re-reading https://jenkins.io/blog/2016/11/21/gc-tuning/

Signed-off-by: Holger Levsen <holger@layer-acht.org>

bin/reproducible_common.sh

@@ -857,6 +857,104 @@ create_debian_png_from_table() {
 	rm ${TABLE[$1]}.csv
 }
 
+#
+# create the png (and query the db to populate a csv file...) for Arch Linux
+#
+create_archlinux_png_from_table() {
+	echo "Checking whether to update $2..."
+	# $1 = id of the stats table
+	# $2 = image file name
+	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
+	# prepare query
+	WHERE_EXTRA="WHERE suite = '$SUITE'"
+	if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then
+		# TABLE[0+2] have a architecture column:
+		WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'"
+	fi
+	# run query
+	if [ $1 -eq 1 ] ; then
+		# not sure if it's worth to generate the following query...
+		WHERE_EXTRA="AND architecture='$ARCH'"
+
+		# This query became much more obnoxious when gaining
+		# compatibility with postgres
+		query_to_csv "SELECT stats.datum,
+			 COALESCE(reproducible_stretch,0) AS reproducible_stretch,
+			 COALESCE(reproducible_buster,0) AS reproducible_buster,
+			 COALESCE(reproducible_unstable,0) AS reproducible_unstable,
+			 COALESCE(reproducible_experimental,0) AS reproducible_experimental,
+			 COALESCE(FTBR_stretch,0) AS FTBR_stretch,
+			 COALESCE(FTBR_buster,0) AS FTBR_buster,
+			 COALESCE(FTBR_unstable,0) AS FTBR_unstable,
+			 COALESCE(FTBR_experimental,0) AS FTBR_experimental,
+			 COALESCE(FTBFS_stretch,0) AS FTBFS_stretch,
+			 COALESCE(FTBFS_buster,0) AS FTBFS_buster,
+			 COALESCE(FTBFS_unstable,0) AS FTBFS_unstable,
+			 COALESCE(FTBFS_experimental,0) AS FTBFS_experimental,
+			 COALESCE(other_stretch,0) AS other_stretch,
+			 COALESCE(other_buster,0) AS other_buster,
+			 COALESCE(other_unstable,0) AS other_unstable,
+			 COALESCE(other_experimental,0) AS other_experimental
+			FROM (SELECT s.datum,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
+			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch,
+			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster,
+			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable,
+			 (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental
+			 FROM stats_builds_per_day AS s GROUP BY s.datum) as stats
+			ORDER BY datum" >> ${TABLE[$1]}.csv
+	elif [ $1 -eq 2 ] ; then
+		# just make a graph of the oldest reproducible build (ignore FTBFS and FTBR)
+		query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+	else
+		query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+	fi
+	# this is a gross hack: normally we take the number of colors a table should have...
+	#  for the builds_age table we only want one color, but different ones, so this hack:
+	COLORS=${COLOR[$1]}
+	if [ $1 -eq 2 ] ; then
+		case "$SUITE" in
+			stretch)	COLORS=40 ;;
+			buster)		COLORS=41 ;;
+			unstable)	COLORS=42 ;;
+			experimental)	COLORS=43 ;;
+		esac
+	fi
+	local WIDTH=1920
+	local HEIGHT=960
+	# only generate graph if the query returned data
+	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
+		echo "Updating $2..."
+		DIR=$(dirname $2)
+		mkdir -p $DIR
+		echo "Generating $2."
+		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT
+		mv $2 $ARCHBASE/$DIR
+		[ "$DIR" = "." ] || rmdir $(dirname $2)
+	# create empty dummy png if there havent been any results ever
+	elif [ ! -f $ARCHBASE/$DIR/$(basename $2) ] ; then
+		DIR=$(dirname $2)
+		mkdir -p $DIR
+		echo "Creating $2 dummy."
+		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
+		mv $2 $ARCHBASE/$DIR
+		[ "$DIR" = "." ] || rmdir $(dirname $2)
+	fi
+	rm ${TABLE[$1]}.csv
+}
+
+
 find_in_buildlogs() {
     egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null
 }

bin/reproducible_html_archlinux.sh

@@ -437,6 +437,15 @@ if [ -z "$1" ] ; then
 	ARCHLINUX_NR_UNKNOWN=0
 	WIDTH=1920
 	HEIGHT=960
+	# variables related to the stats we update
+	# FIELDS[0]="datum, reproducible, FTBR, FTBFS, other, untested" # FIXME: for this Arch Linux still uses a .csv file...
+	FIELDS[1]="datum"
+	for i in reproducible FTBR FTBFS other ; do
+	        for j in $SUITES ; do
+	                FIELDS[1]="${FIELDS[1]}, ${i}_${j}"
+	        done
+	done
+	FIELDS[2]="datum, oldest"
 
 	repostats
 	dashboard_page

hosts/jenkins/etc/default/jenkins

@@ -17,7 +17,7 @@ JAVA_ARGS="-Djava.awt.headless=true"
 #JAVA_ARGS="-Xmx256m"
 #JAVA_ARGS="-Djava.net.preferIPv4Stack=true" # make jenkins listen on IPv4 address
 # See some recommendation from https://jenkins.io/blog/2016/11/21/gc-tuning/
-JAVA_ARGS="$JAVA_ARGS -Xms4G -Xmx8G"  # heap settings
+JAVA_ARGS="$JAVA_ARGS -Xms8G -Xmx24G"  # heap settings
 JAVA_ARGS="$JAVA_ARGS -server -XX:+AlwaysPreTouch"
 # store heap dumps in case of OOM, to investigate it later
 JAVA_ARGS="$JAVA_ARGS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$JENKINS_HOME/heapDumps/"
@@ -30,8 +30,8 @@ JAVA_ARGS="$JAVA_ARGS -XX:+PrintGCCause -XX:+PrintTenuringDistribution -XX:+Prin
 JAVA_ARGS="$JAVA_ARGS -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication"
 JAVA_ARGS="$JAVA_ARGS -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1"
 # h01ger trying stuff
-JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops"
-MALLOC_ARENA_MAX=2
+JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops -XX:+UseCompressedClassPointers"
+export MALLOC_ARENA_MAX=2
 
 # make jenkins listen on IPv4 address
 #JAVA_ARGS="-Djava.net.preferIPv4Stack=true"