reproducible: add two new graphs showing the reproducible usertaggsed bugs except the ftbfs ones

TODO

@@ -160,7 +160,6 @@ This is about Debian, below are more todo entries for other projects…
 ** link howto on each coreboot/openwrt/netbsd/freebsd page
 ** pkg sets are still amd64 only atm… (and there is 404 link to the armhf page)
 ** bin/_html_indexes.py: bugs = get_bugs() # this variable should not be global, else merely importing _html_indexes always queries UDD
-** new graph: looking at https://reproducible.debian.net/stats_bugs.png i wonder if it would be worthwhile to have https://reproducible.debian.net/stats_bugs_state.png but without the ftbfs tagged bugs… i'd hope that this graph is a.) new+interesting and b.) showing progress
 
 * lesser prio
 ** check that build nodes have different amount of cores, so we dont need to run the 2nd build with NUM_CPU-1

bin/reproducible_common.sh

@@ -79,6 +79,8 @@ TABLE[4]=stats_notes
 TABLE[5]=stats_issues
 TABLE[6]=stats_meta_pkg_state
 TABLE[7]=stats_bugs_state
+TABLE[8]=stats_bugs_sin_ftbfs
+TABLE[9]=stats_bugs_sin_ftbfs_state
 
 # known package sets
 META_PKGSET[1]="essential"
@@ -574,8 +576,8 @@ create_png_from_table() {
 	else
 		WHERE2_EXTRA=""
 	fi
-	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then
-		# TABLE[3+4+5] don't have a suite column:
+	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] || [ $1 -eq 8 ] ; then
+		# TABLE[3+4+5] don't have a suite column: (and TABLE[8] (and 9) is faked, based on 3)
 		WHERE_EXTRA=""
 	elif [ $1 -eq 6 ] ; then
 		# 6 is special too:
@@ -616,6 +618,10 @@ create_png_from_table() {
 		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
 	elif [ $1 -eq 7 ] ; then
 		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
+	elif [ $1 -eq 8 ] ; then
+		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[3]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+	elif [ $1 -eq 9 ] ; then
+		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $REPRODUCIBLE_DONE, $REPRODUCIBLE_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
 	else
 		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
 	fi

bin/reproducible_html_dashboard.sh

@@ -43,6 +43,25 @@ for TAG in $USERTAGS ; do
 done
 SUM_DONE="$SUM_DONE)"
 SUM_OPEN="$SUM_OPEN)"
+FIELDS[8]="datum "
+for TAG in $USERTAGS ; do
+	if [ "$TAG" = "ftbfs" ] ; then
+		continue
+	fi
+	FIELDS[8]="${FIELDS[8]}, open_$TAG, done_$TAG"
+done
+FIELDS[9]="datum, done_bugs, open_bugs"
+REPRODUCIBLE_DONE="(0"
+REPRODUCIBLE_OPEN="(0"
+for TAG in $USERTAGS ; do
+	if [ "$TAG" = "ftbfs" ] ; then
+		continue
+	fi
+	REPRODUCIBLE_DONE="$REPRODUCIBLE_DONE+done_$TAG"
+	REPRODUCIBLE_OPEN="$REPRODUCIBLE_OPEN+open_$TAG"
+done
+REPRODUCIBLE_DONE="$REPRODUCIBLE_DONE)"
+REPRODUCIBLE_OPEN="$REPRODUCIBLE_OPEN)"
 COLOR[0]=5
 COLOR[1]=12
 COLOR[2]=1
@@ -50,11 +69,15 @@ COLOR[3]=32
 COLOR[4]=1
 COLOR[5]=1
 COLOR[7]=2
+COLOR[8]=30
+COLOR[9]=2
 MAINLABEL[1]="Amount of packages built each day"
-MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org"
+MAINLABEL[3]="Bugs (with all usertags) for user reproducible-builds@lists.alioth.debian.org"
 MAINLABEL[4]="Packages which have notes"
 MAINLABEL[5]="Identified issues"
-MAINLABEL[7]="Open and closed bugs"
+MAINLABEL[7]="Open and closed bugs (with all usertags)"
+MAINLABEL[8]="Bugs (with all usertags except 'ftbfs') for user reproducible-builds@lists.alioth.debian.org"
+MAINLABEL[9]="Open and closed bugs (with all usertags except tagged 'ftbfs')"
 YLABEL[0]="Amount (total)"
 YLABEL[1]="Amount (per day)"
 YLABEL[2]="Age in days"
@@ -62,6 +85,8 @@ YLABEL[3]="Amount of bugs"
 YLABEL[4]="Amount of packages"
 YLABEL[5]="Amount of issues"
 YLABEL[7]="Amount of bugs open / closed"
+YLABEL[8]="Amount of bugs"
+YLABEL[9]="Amount of bugs open / closed"
 
 #
 # update package + build stats
@@ -199,10 +224,11 @@ update_bug_stats() {
 			echo "Updating ${PACKAGES_DB} with bug stats for $DATE."
 			sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL"
 			# force regeneration of the image
-			echo "Touching ${TABLE[3]}.png..."
-			touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[3]}.png
-			echo "Touching ${TABLE[7]}.png..."
-			touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[7]}.png
+			local i=0
+			for i in 3 7 8 9 ; do
+				echo "Touching ${TABLE[$i]}.png..."
+				touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[$i]}.png
+			done
 		fi
 	fi
 }
@@ -416,7 +442,7 @@ create_dashboard_page() {
 	write_usertag_table
 	write_page "</p><p style=\"clear:both;\">"
 	# do other global graphs
-	for i in 3 7 4 5 ; do
+	for i in 3 7 8 9 4 5 ; do
 		write_page " <a href=\"/${TABLE[$i]}.png\"><img src=\"/${TABLE[$i]}.png\" class="halfview" alt=\"${MAINLABEL[$i]}\"></a>"
 		# redo pngs once a day
 		if [ ! -f $BASE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then