reproducible_html_archlinux.sh 7.61 KB
Newer Older
1 2
#!/bin/bash

3
# Copyright 2014-2018 Holger Levsen <holger@layer-acht.org>
4
#                2015 anthraxx <levente@leventepolyak.net>
5 6
# released under the GPLv=2

7
DEBUG=false
8 9 10
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

11
# common code
12 13 14
. /srv/jenkins/bin/reproducible_common.sh

#
15
# analyse results to create the webpage
16
#
17
echo "$(date -u) - starting to analyse build results."
18
DATE=$(date -u +'%Y-%m-%d')
19
YESTERDAY=$(date '+%Y-%m-%d' -d "-1 day")
20
MEMBERS_FTBFS="0 1 2 3 4"
21
MEMBERS_DEPWAIT="0 1"
22
MEMBERS_404="0 1 2 3 4 5 6 7 8 9 A"
23
MEMBERS_FTBR="0 1 2"
24 25
HTML_BUFFER=$(mktemp -t archlinuxrb-html-XXXXXXXX)
HTML_REPOSTATS=$(mktemp -t archlinuxrb-html-XXXXXXXX)
26 27 28 29 30 31 32
ARCHLINUX_TOTAL=0
ARCHLINUX_TESTED=0
ARCHLINUX_NR_FTBFS=0
ARCHLINUX_NR_FTBR=0
ARCHLINUX_NR_DEPWAIT=0
ARCHLINUX_NR_404=0
ARCHLINUX_NR_GOOD=0
33
ARCHLINUX_NR_BLACKLISTED=0
34
ARCHLINUX_NR_UNKNOWN=0
35 36
WIDTH=1920
HEIGHT=960
37
for REPOSITORY in $ARCHLINUX_REPOS ; do
38 39
	echo "$(date -u) - starting to analyse build results for '$REPOSITORY'."
	# prepare stats per repository
40 41 42 43 44 45 46 47 48
	TOTAL=$(cat ${ARCHLINUX_PKGS}_$REPOSITORY | wc -l)
	TESTED=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c ^ || true )
	NR_GOOD=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c GOOD || true )
	NR_FTBR=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c FTBR || true )
	NR_FTBFS=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c FTBFS || true )
	NR_DEPWAIT=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c DEPWAIT || true )
	NR_404=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c 404 || true )
	NR_BLACKLISTED=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c BLACKLISTED || true )
	NR_UNKNOWN=$(cat $ARCHBASE/$REPOSITORY/*/pkg.state | grep -c UNKNOWN || true )
49
	PERCENT_TOTAL=$(echo "scale=1 ; ($TESTED*100/$TOTAL)" | bc)
50
	if [ $(echo $PERCENT_TOTAL/1|bc) -lt 99 ] ; then
51
		NR_TESTED="$TESTED <span style=\"font-size:0.8em;\">(tested $PERCENT_TOTAL% of $TOTAL)</span>"
52 53 54 55 56
	else
		NR_TESTED=$TESTED
	fi
	echo "     <tr>" >> $HTML_REPOSTATS
	echo "      <td>$REPOSITORY</td><td>$NR_TESTED</td>" >> $HTML_REPOSTATS
57
	for i in $NR_GOOD $NR_FTBR $NR_FTBFS $NR_DEPWAIT $NR_404 $NR_BLACKLISTED $NR_UNKNOWN ; do
58
		PERCENT_i=$(echo "scale=1 ; ($i*100/$TESTED)" | bc)
59
		if [ "$PERCENT_i" != "0" ] || [ "$i" != "0" ] ; then
60 61 62 63 64 65
			echo "      <td>$i ($PERCENT_i%)</td>" >> $HTML_REPOSTATS
		else
			echo "      <td>$i</td>" >> $HTML_REPOSTATS
		fi
	done
	echo "     </tr>" >> $HTML_REPOSTATS
66
	#
67
	# write csv file for $REPOSITORY
68
	#
69
	if [ ! -f $ARCHBASE/$REPOSITORY.csv ] ; then
70
		echo '; date, reproducible, unreproducible, ftbfs, depwait, download problems, untested' > $ARCHBASE/$REPOSITORY.csv
71
	fi
72
	if ! grep -q $YESTERDAY $ARCHBASE/$REPOSITORY.csv ; then
73
		let REAL_UNKNOWN=$TOTAL-$NR_GOOD-$NR_FTBR-$NR_FTBFS-$NR_DEPWAIT-$NR_404 || true
74
		echo $YESTERDAY,$NR_GOOD,$NR_FTBR,$NR_FTBFS,$NR_DEPWAIT,$NR_404,$REAL_UNKNOWN >> $ARCHBASE/$REPOSITORY.csv
75
	fi
76 77 78 79 80
	IMAGE=$ARCHBASE/$REPOSITORY.png
	if [ ! -f $IMAGE ] || [ $ARCHBASE/$REPOSITORY.csv -nt $IMAGE ] ; then
		echo "Updating $IMAGE..."
		/srv/jenkins/bin/make_graph.py $ARCHBASE/$REPOSITORY.csv $IMAGE 6 "Reproducibility status for Arch Linux packages in $REPOSITORY" "Amount (total)" $WIDTH $HEIGHT
	fi
81
	#
82
	# prepare ARCHLINUX totals
83
	#
84 85 86 87 88 89 90 91
	set +e
	let ARCHLINUX_TOTAL+=$TOTAL
	let ARCHLINUX_TESTED+=$TESTED
	let ARCHLINUX_NR_FTBFS+=$NR_FTBFS
	let ARCHLINUX_NR_FTBR+=$NR_FTBR
	let ARCHLINUX_NR_DEPWAIT+=$NR_DEPWAIT
	let ARCHLINUX_NR_404+=$NR_404
	let ARCHLINUX_NR_GOOD+=$NR_GOOD
92
	let ARCHLINUX_NR_BLACKLISTED+=$NR_BLACKLISTED
93 94
	let ARCHLINUX_NR_UNKNOWN+=$NR_UNKNOWN
	set -e
95
done
96 97 98
# prepare stats per repository
ARCHLINUX_PERCENT_TOTAL=$(echo "scale=1 ; ($ARCHLINUX_TESTED*100/$ARCHLINUX_TOTAL)" | bc)
if [ $(echo $ARCHLINUX_PERCENT_TOTAL/1|bc) -lt 99 ] ; then
99
	NR_TESTED="$ARCHLINUX_TESTED <span style=\"font-size:0.8em;\">(tested $ARCHLINUX_PERCENT_TOTAL% of $ARCHLINUX_TOTAL)</span>"
100 101 102 103
else
	NR_TESTED=$ARCHLINUX_TESTED
fi
echo "     <tr>" >> $HTML_REPOSTATS
104
echo "      <td><b>all combined</b></td><td>$NR_TESTED</td>" >> $HTML_REPOSTATS
105
for i in $ARCHLINUX_NR_GOOD $ARCHLINUX_NR_FTBR $ARCHLINUX_NR_FTBFS $ARCHLINUX_NR_DEPWAIT $ARCHLINUX_NR_404 $ARCHLINUX_NR_BLACKLISTED $ARCHLINUX_NR_UNKNOWN ; do
106
	PERCENT_i=$(echo "scale=1 ; ($i*100/$ARCHLINUX_TESTED)" | bc)
107
	if [ "$PERCENT_i" != "0" ] || [ "$i" != "0" ] ; then
108
		echo "      <td>$i ($PERCENT_i%)</td>" >> $HTML_REPOSTATS
109 110 111 112 113
	else
		echo "      <td>$i</td>" >> $HTML_REPOSTATS
	fi
done
echo "     </tr>" >> $HTML_REPOSTATS
114 115 116 117

#
# write csv file for totals
#
118
if [ ! -f $ARCHBASE/archlinux.csv ] ; then
119
	echo '; date, reproducible, unreproducible, ftbfs, depwait, download problems, untested' > $ARCHBASE/archlinux.csv
120
fi
121
if ! grep -q $YESTERDAY $ARCHBASE/archlinux.csv ; then
122
	let ARCHLINUX_REAL_UNKNOWN=$ARCHLINUX_TOTAL-$ARCHLINUX_NR_GOOD-$ARCHLINUX_NR_FTBR-$ARCHLINUX_NR_FTBFS-$ARCHLINUX_NR_DEPWAIT-$ARCHLINUX_NR_404 || true
123
	echo $YESTERDAY,$ARCHLINUX_NR_GOOD,$ARCHLINUX_NR_FTBR,$ARCHLINUX_NR_FTBFS,$ARCHLINUX_NR_DEPWAIT,$ARCHLINUX_NR_404,$ARCHLINUX_REAL_UNKNOWN >> $ARCHBASE/archlinux.csv
124
fi
125 126 127 128
IMAGE=$ARCHBASE/archlinux.png
if [ ! -f $IMAGE ] || [ $ARCHBASE/archlinux.csv -nt $IMAGE ] ; then
	echo "Updating $IMAGE..."
	/srv/jenkins/bin/make_graph.py $ARCHBASE/archlinux.csv $IMAGE 6 "Reproducibility status for all tested Arch Linux packages" "Amount (total)" $WIDTH $HEIGHT
129
	irc_message archlinux-reproducible "Daily graphs on $REPRODUCIBLE_URL/archlinux/ updated, $(echo "scale=1 ; ($ARCHLINUX_NR_GOOD*100/$ARCHLINUX_TESTED)" | bc)% reproducible packages in our current test framework."
130
fi
131

132 133 134 135 136 137 138 139 140 141 142
#
# write out the actual webpage
#
cd $ARCHBASE
PAGE=archlinux.html
echo "$(date -u) - starting to build $PAGE"
cat > $PAGE <<- EOF
<!DOCTYPE html>
<html lang="en-US">
  <head>
    <meta charset="UTF-8">
143
    <title>Reproducible Arch Linux ?!</title>
144 145 146 147 148 149 150 151 152 153 154 155
    <link rel='stylesheet' href='global.css' type='text/css' media='all' />
  </head>
  <body>
    <div id="archnavbar">
	    <div id="logo"></div>
    </div>
    <div class="content">
      <h1>Reproducible Arch Linux?!</h1>
      <div class="page-content">

EOF
write_page_intro 'Arch Linux'
156
write_variation_table 'Arch Linux'
157
write_page "    <table><tr><th>repository</th><th>all source packages</th><th>reproducible packages</th><th>unreproducible packages</th><th>packages failing to build</th><th>packages in depwait state</th><th>packages download problems</th><th>blacklisted</th><th>unknown state</th></tr>"
158 159 160
cat $HTML_REPOSTATS >> $PAGE
rm $HTML_REPOSTATS > /dev/null
write_page "    </table>"
161 162 163 164 165 166 167 168
# include graphs
write_page '<p style="clear:both;">'
for REPOSITORY in $ARCHLINUX_REPOS ; do
	write_page "<a href=\"/archlinux/$REPOSITORY.png\"><img src=\"/archlinux/$REPOSITORY.png\" class=\"overview\" alt=\"$REPOSITORY stats\"></a>"
done
write_page '</p><p style="clear:both;"><center>'
write_page "<a href=\"/archlinux/archlinux.png\"><img src=\"/archlinux/archlinux.png\" alt=\"total Arch Linux stats\"></a></p>"
# packages table header
169
write_page "    <table><tr><th>repository</th><th>source package</th><th>version</th><th>test result</th><th>test date<br />test duration</th><th>1st build log<br />2nd build log</th></tr>"
170
# output all HTML snipplets
171
for i in UNKNOWN $(for j in $MEMBERS_404 ; do echo 404_$j ; done) BLACKLISTED $(for j in $MEMBERS_DEPWAIT ; do echo DEPWAIT_$j ; done) $(for j in $MEMBERS_FTBFS ; do echo FTBFS_$j ; done) $(for j in $MEMBERS_FTBR ; do echo FTBR_$j ; done) GOOD ; do
172
	for REPOSITORY in $ARCHLINUX_REPOS ; do
173
		grep -l $i $REPOSITORY/*/pkg.state | sort -u | sed -s 's#\.state$#.html#g' | xargs -r cat >> $PAGE 2>/dev/null || true
174
	done
175
done
176 177
write_page "    </table>"
write_page "</div></div>"
178
write_page_footer 'Arch Linux'
179
echo "$(date -u) - enjoy $REPRODUCIBLE_URL/archlinux/$PAGE"
180

181 182
echo "$(date -u) - Sleeping 5min now to prevent immediate restart of this job…"
sleep 5m
183

184
# vim: set sw=0 noet :