diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 0d1beea008ee964ede6b94b6b1f05a01afb630ad..487de18970ec261bad18f726f19e3d7727e922d7 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -857,6 +857,104 @@ create_debian_png_from_table() { rm ${TABLE[$1]}.csv } +# +# create the png (and query the db to populate a csv file...) for Arch Linux +# +create_archlinux_png_from_table() { + echo "Checking whether to update $2..." + # $1 = id of the stats table + # $2 = image file name + echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv + # prepare query + WHERE_EXTRA="WHERE suite = '$SUITE'" + if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then + # TABLE[0+2] have a architecture column: + WHERE_EXTRA="$WHERE_EXTRA AND architecture = '$ARCH'" + fi + # run query + if [ $1 -eq 1 ] ; then + # not sure if it's worth to generate the following query... + WHERE_EXTRA="AND architecture='$ARCH'" + + # This query became much more obnoxious when gaining + # compatibility with postgres + query_to_csv "SELECT stats.datum, + COALESCE(reproducible_stretch,0) AS reproducible_stretch, + COALESCE(reproducible_buster,0) AS reproducible_buster, + COALESCE(reproducible_unstable,0) AS reproducible_unstable, + COALESCE(reproducible_experimental,0) AS reproducible_experimental, + COALESCE(FTBR_stretch,0) AS FTBR_stretch, + COALESCE(FTBR_buster,0) AS FTBR_buster, + COALESCE(FTBR_unstable,0) AS FTBR_unstable, + COALESCE(FTBR_experimental,0) AS FTBR_experimental, + COALESCE(FTBFS_stretch,0) AS FTBFS_stretch, + COALESCE(FTBFS_buster,0) AS FTBFS_buster, + COALESCE(FTBFS_unstable,0) AS FTBFS_unstable, + COALESCE(FTBFS_experimental,0) AS FTBFS_experimental, + COALESCE(other_stretch,0) AS other_stretch, + COALESCE(other_buster,0) AS other_buster, + COALESCE(other_unstable,0) AS other_unstable, + COALESCE(other_experimental,0) AS other_experimental + FROM (SELECT s.datum, + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA),0) AS reproducible_stretch, + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA),0) AS reproducible_buster, + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable, + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental, + (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBR_stretch, + (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBR_buster, + (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBR_unstable, + (SELECT e.FTBR FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBR_experimental, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS FTBFS_stretch, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS FTBFS_buster, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='stretch' $WHERE_EXTRA) AS other_stretch, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='buster' $WHERE_EXTRA) AS other_buster, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental + FROM stats_builds_per_day AS s GROUP BY s.datum) as stats + ORDER BY datum" >> ${TABLE[$1]}.csv + elif [ $1 -eq 2 ] ; then + # just make a graph of the oldest reproducible build (ignore FTBFS and FTBR) + query_to_csv "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv + else + query_to_csv "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv + fi + # this is a gross hack: normally we take the number of colors a table should have... + # for the builds_age table we only want one color, but different ones, so this hack: + COLORS=${COLOR[$1]} + if [ $1 -eq 2 ] ; then + case "$SUITE" in + stretch) COLORS=40 ;; + buster) COLORS=41 ;; + unstable) COLORS=42 ;; + experimental) COLORS=43 ;; + esac + fi + local WIDTH=1920 + local HEIGHT=960 + # only generate graph if the query returned data + if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then + echo "Updating $2..." + DIR=$(dirname $2) + mkdir -p $DIR + echo "Generating $2." + /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT + mv $2 $ARCHBASE/$DIR + [ "$DIR" = "." ] || rmdir $(dirname $2) + # create empty dummy png if there havent been any results ever + elif [ ! -f $ARCHBASE/$DIR/$(basename $2) ] ; then + DIR=$(dirname $2) + mkdir -p $DIR + echo "Creating $2 dummy." + convert -size 1920x960 xc:#aaaaaa -depth 8 $2 + mv $2 $ARCHBASE/$DIR + [ "$DIR" = "." ] || rmdir $(dirname $2) + fi + rm ${TABLE[$1]}.csv +} + + find_in_buildlogs() { egrep -q "$1" $ARCHLINUX_PKG_PATH/build1.log $ARCHLINUX_PKG_PATH/build2.log 2>/dev/null } diff --git a/bin/reproducible_html_archlinux.sh b/bin/reproducible_html_archlinux.sh index c034ab9a845872855e7fd399a3abd657b26408bf..67c8c6224bf34d8bbde4badbeab00911532a2064 100755 --- a/bin/reproducible_html_archlinux.sh +++ b/bin/reproducible_html_archlinux.sh @@ -437,6 +437,15 @@ if [ -z "$1" ] ; then ARCHLINUX_NR_UNKNOWN=0 WIDTH=1920 HEIGHT=960 + # variables related to the stats we update + # FIELDS[0]="datum, reproducible, FTBR, FTBFS, other, untested" # FIXME: for this Arch Linux still uses a .csv file... + FIELDS[1]="datum" + for i in reproducible FTBR FTBFS other ; do + for j in $SUITES ; do + FIELDS[1]="${FIELDS[1]}, ${i}_${j}" + done + done + FIELDS[2]="datum, oldest" repostats dashboard_page diff --git a/hosts/jenkins/etc/default/jenkins b/hosts/jenkins/etc/default/jenkins index bb2a1e3102a146836598c0696df1ff5939ddc296..76cf27358c5fd113fd4235025ec15adfd7a07808 100644 --- a/hosts/jenkins/etc/default/jenkins +++ b/hosts/jenkins/etc/default/jenkins @@ -17,7 +17,7 @@ JAVA_ARGS="-Djava.awt.headless=true" #JAVA_ARGS="-Xmx256m" #JAVA_ARGS="-Djava.net.preferIPv4Stack=true" # make jenkins listen on IPv4 address # See some recommendation from https://jenkins.io/blog/2016/11/21/gc-tuning/ -JAVA_ARGS="$JAVA_ARGS -Xms4G -Xmx8G" # heap settings +JAVA_ARGS="$JAVA_ARGS -Xms8G -Xmx24G" # heap settings JAVA_ARGS="$JAVA_ARGS -server -XX:+AlwaysPreTouch" # store heap dumps in case of OOM, to investigate it later JAVA_ARGS="$JAVA_ARGS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$JENKINS_HOME/heapDumps/" @@ -30,8 +30,8 @@ JAVA_ARGS="$JAVA_ARGS -XX:+PrintGCCause -XX:+PrintTenuringDistribution -XX:+Prin JAVA_ARGS="$JAVA_ARGS -XX:+UseG1GC -XX:+ExplicitGCInvokesConcurrent -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication" JAVA_ARGS="$JAVA_ARGS -XX:+UnlockExperimentalVMOptions -XX:G1NewSizePercent=20 -XX:+UnlockDiagnosticVMOptions -XX:G1SummarizeRSetStatsPeriod=1" # h01ger trying stuff -JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops" -MALLOC_ARENA_MAX=2 +JAVA_ARGS="$JAVA_ARGS -XX:+UseCompressedOops -XX:+UseCompressedClassPointers" +export MALLOC_ARENA_MAX=2 # make jenkins listen on IPv4 address #JAVA_ARGS="-Djava.net.preferIPv4Stack=true"