reproducible_html_repository_comparison.sh 8.79 KB
Newer Older
1 2 3 4 5
#!/bin/bash

# Copyright 2015 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2

6
DEBUG=false
7 8 9 10 11
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
12 13
 
# overwrite defaults as we need this order here
14
ARCHS="amd64 armhf"
15

16
VIEW=repositories
17
PAGE=index_${VIEW}.html
Holger Levsen's avatar
Holger Levsen committed
18
SOURCES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
19
PACKAGES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
20
TMPFILE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
21 22
TABLE_TODO=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
TABLE_DONE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
23

24
MODIFIED_IN_SID=0
25
MODIFIED_IN_EXP=0
26
BINNMUS_NEEDED=0
27

28 29 30 31
write_row() {
        echo "$1" >> $ROW
}

32
ARCH="amd64"
33
SUITE="unstable"
34
echo "$(date) - starting to write $PAGE page."
Holger Levsen's avatar
Holger Levsen committed
35
write_page_header $VIEW "Comparison between the reproducible builds apt repository and regular Debian suites"
36
write_page "<p>These source packages are different from unstable in our apt repository on alioth. They are available for <a href=\"https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain#Usage_example\">testing using these sources.lists</a> entries:<pre>"
37 38 39
write_page "deb http://reproducible.alioth.debian.org/debian/ ./"
write_page "deb-src http://reproducible.alioth.debian.org/debian/ ./"
write_page "</pre></p>"
40
write_page "<p><table><tr><th>package</th><th>git repo</th><th>PTS link</th><th>usertagged bug</th><th>old versions in our repo<br />(needed for reproducing old builds)</th><th>version in our repo<br />(available binary packages per architecture)</th><th>version in 'testing'</th><th>version in 'unstable'</th><th>version in 'experimental'</th></tr>"
41

Holger Levsen's avatar
Holger Levsen committed
42
curl http://reproducible.alioth.debian.org/debian/Sources > $SOURCES
43
curl http://reproducible.alioth.debian.org/debian/Packages > $PACKAGES
Holger Levsen's avatar
Holger Levsen committed
44
SOURCEPKGS=$(grep-dctrl -n -s Package -r -FPackage . $SOURCES | sort -u)
Holger Levsen's avatar
Holger Levsen committed
45
for PKG in $SOURCEPKGS ; do
46 47 48 49 50 51
	echo "Processing $PKG..."
	if [ "${PKG:0:3}" = "lib" ] ; then
		PREFIX=${PKG:0:4}
	else
		PREFIX=${PKG:0:1}
	fi
Holger Levsen's avatar
Holger Levsen committed
52
	VERSIONS=$(grep-dctrl -n -s version -S $PKG $SOURCES|sort -u)
53
	CRUFT=""
54
	BET=""
55 56
	OBSOLETE_IN_SID=false
	OBSOLETE_IN_TESTING=false
57
	OBSOLETE_IN_EXP=false
58 59 60
	#
	# gather versions of a package
	#
61 62 63 64 65 66 67 68
	for VERSION in ${VERSIONS} ; do
		if [ "$BET" = "" ] ; then
			BET=${VERSION}
			continue
		elif dpkg --compare-versions "$BET" lt "${VERSION}"  ; then
			BET=${VERSION}
		fi
	done
69
	SID=$(rmadison -s unstable $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo)
70 71
	for VERSION in ${VERSIONS} ; do
		if [ "${VERSION}" != "$BET" ] ; then
Holger Levsen's avatar
Holger Levsen committed
72
			CRUFT="$CRUFT ${VERSION}"
73 74
		fi
	done
75 76
	TESTING=$(rmadison -s testing $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo)
	EXPERIMENTAL=$(rmadison -s experimental $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo)
77
	#
78
	# format output
79 80
	#
	CSID=""
81 82
	CTEST=""
	CEXP=""
83 84
	if [ ! -z "$TESTING" ] ; then
		for i in $TESTING ; do
85
			if dpkg --compare-versions "$i" gt "$BET" ; then
86
				CTEST="$CTEST<span class=\"green\">$i</span><br />"
87
				OBSOLETE_IN_TESTING=true
88 89
				OBSOLETE_IN_SID=true
				OBSOLETE_IN_EXP=true
90 91 92
			else
				CTEST="$CTEST$i<br />"
			fi
93 94
		done
	fi
Holger Levsen's avatar
Holger Levsen committed
95 96
	if [ ! -z "$EXPERIMENTAL" ] ; then
		for i in $EXPERIMENTAL ; do
97 98
			if dpkg --compare-versions "$i" gt "$BET" ; then
				CEXP="$CEXP<a href=\"https://tracker.debian.org/media/packages/$PREFIX/$PKG/changelog-$i\">$i</a><br />"
99 100
				OBSOLETE_IN_EXP=true
		else
101 102
				CEXP="$CEXP$i<br />"
			fi
103
		done
104
	fi
105 106 107 108 109 110 111
	for i in $SID ; do
		if dpkg --compare-versions "$i" gt "$BET" ; then
			CSID="$CSID<a href=\"https://tracker.debian.org/media/packages/$PREFIX/$PKG/changelog-$i\">$i</a><br />"
			if [ ! -z "$BET" ] ; then
				CRUFT="$BET $CRUFT"
				BET=""
				OBSOLETE_IN_SID=true
112
				OBSOLETE_IN_EXP=true
113 114 115 116 117
			fi
		else
			CSID="$CSID$i<br />"
		fi
	done
118
	CBINARIES=""
119
	if [ ! -z "$BET" ] ; then
120 121
		ONLYALL=true
		for ARCH in all ${ARCHS} ; do
122
			i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)"
123 124 125 126 127 128
			if [ "$ARCH" != "all" ] && [ ! -z "$i" ] ; then
				ONLYALL=false
			fi
			echo "$ARCH: $i"
		done
		for ARCH in all ${ARCHS} ; do
129
			i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)"
130
			if [ ! -z "$i" ] ; then
131 132
				i="$ARCH: $i"
			elif [ -z "$i" ] && [ "$ARCH" != "all" ] && ! $ONLYALL ; then
133
				i="<span class=\"red\">no binaries for $ARCH</span>"
134
				let "BINNMUS_NEEDED+=1"
135
			fi
136
			CBINARIES="$CBINARIES<br />$i"
137
		done
138
		BET="<span class=\"green\">$BET</span>"
139
		ROW=$TABLE_TODO
140 141
	else
		BET="&nbsp;"
142
		ROW=$TABLE_DONE
143 144 145 146
	fi
	if [ ! -z "$CRUFT" ] ; then
		CRUFT="$(echo $CRUFT|sed 's# #<br />#g')"
	fi
147 148 149
	#
	# write output
	#
150 151
	write_row "<tr><td><pre>src:$PKG</pre></td>"
	write_row " <td>"
152
	GIT="$PKG.git"
153
	case $PKG in
154 155 156
		debbindiff)
			URL="http://anonscm.debian.org/cgit/reproducible/diffoscope.git"
			GIT="diffoscope.git" ;;
157
		strip-nondeterminism|diffoscope|disorderfs)
158
			URL="http://anonscm.debian.org/cgit/reproducible/$GIT" ;;
159
		*)
160
			URL="http://anonscm.debian.org/cgit/reproducible/$GIT/?h=pu/reproducible_builds" ;;
161
	esac
Holger Levsen's avatar
Holger Levsen committed
162 163
	curl $URL > $TMPFILE
	if [ "$(grep "'error'>No repositories found" $TMPFILE 2>/dev/null)" ] ; then
164
		write_row "$URL<br /><span class=\"red\">(no git repository found)</span>"
Holger Levsen's avatar
Holger Levsen committed
165
	elif [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then
166
		URL="http://anonscm.debian.org/cgit/reproducible/$GIT/?h=merged/reproducible_builds"
Holger Levsen's avatar
Holger Levsen committed
167 168
		curl $URL > $TMPFILE
		if [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then
169
			if ! $OBSOLETE_IN_SID ; then
170
				write_row "<a href=\"$URL\">$GIT</a><br /><span class=\"purple\">non-standard branch</span>"
171
			else
172
				write_row "<a href=\"$URL\">$GIT</a><br /><span class=\"green\">non-standard branch</span> (but that is ok, our package ain't used in unstable)"
173 174
			fi
		else
175
			write_row "<a href=\"$URL\">$GIT</a>"
176
			write_row "<br />(<span class=\"green\">merged</span>"
177
			if $OBSOLETE_IN_TESTING ; then
178
				write_row "and available in testing and unstable)"
179
			elif $OBSOLETE_IN_SID ; then
180
				write_row "and available in unstable)"
181
			elif $OBSOLETE_IN_EXP ; then
182
				write_row "and available in experimental)"
183
			fi
184
		fi
185
	else
186
		write_row "<a href=\"$URL\">$GIT</a>"
187
		if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "diffoscope" ] && [ "$PKG" != "debbindiff" ] && [ "$PKG" != "disorderfs" ] ; then
188
			if $OBSOLETE_IN_TESTING && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
189
				write_row "<br />(unused?"
190
				write_row "<br /><span class=\"purple\">Then the branch should probably renamed.</span>)"
191
			elif $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
192
				write_row "<br />(only used in testing, fixed in sid,"
193
				write_row "<br /><span class=\"purple\">branch probably either should be renamed to <em>merged/reproducible_builds</em> or a new upload to our repo is needed?</span>)"
194
			elif $OBSOLETE_IN_EXP ; then
195
				write_row "<br />(only used in testing and unstable, fixed in experimental)"
196
			fi
197 198
		elif [ "$PKG" = "disorderfs" ] ; then
			write_row "<br />(only used to modify the build environment in the 2nd build)"
199
		elif ( [ "$PKG" = "strip-nondeterminism" ] || [ "$PKG" = "diffoscope" ] ) && $OBSOLETE_IN_SID ; then
200
			write_row "<br />(this repo is always used)"
201 202
		elif [ "$PKG" = "debbindiff" ] && $OBSOLETE_IN_SID ; then
			write_row "<br />(debbindiff has been renamed to diffoscope)"
203
		fi
204
	fi
205 206 207
	if ! $OBSOLETE_IN_SID ; then
		let "MODIFIED_IN_SID+=1"
	fi
208 209 210
	if ! $OBSOLETE_IN_EXP ; then
		let "MODIFIED_IN_EXP+=1"
	fi
211 212
	write_row " </td>"
	write_row " <td><a href=\"https://tracker.debian.org/pkg/$PKG\">PTS</a></td>"
213 214 215
	URL="https://bugs.debian.org/cgi-bin/pkgreport.cgi?src=$PKG&users=reproducible-builds@lists.alioth.debian.org&archive=both"
	for TAG in $USERTAGS ; do
		URL="$URL&tag=$TAG"
216
	done
217 218 219 220 221 222 223
	write_row " <td><a href=\"$URL\">bugs</a></td>"
	write_row " <td>$CRUFT</td>"
	write_row " <td>$BET $CBINARIES</td>"
	write_row " <td>$CTEST</td>"
	write_row " <td>$CSID</td>"
	write_row " <td>$CEXP</td>"
	write_row "</tr>"
224 225
	echo "TABLE_TODO: $(ls -la $TABLE_TODO)"
	echo "TABLE_DONE: $(ls -la $TABLE_DONE)"
226
done
227 228 229 230
cat $TABLE_TODO >> $PAGE
write_page "</table></p>"
write_page "<p><table><tr><th>package (obsolete in our repo)</th><th>git repo</th><th>PTS link</th><th>usertagged bug</th><th>old versions in our repo<br />(needed for reproducing old builds)</th><th>version in our repo<br />(available binary packages per architecture)</th><th>version in 'testing'</th><th>version in 'unstable'</th><th>version in 'experimental'</th></tr>"
cat $TABLE_DONE >> $PAGE
231 232 233
write_page "</table></p>"
write_page_footer
publish_page
234
echo "$MODIFIED_IN_SID" > /srv/reproducible-results/modified_in_sid.txt
235
echo "$MODIFIED_IN_EXP" > /srv/reproducible-results/modified_in_exp.txt
236
echo "$BINNMUS_NEEDED" > /srv/reproducible-results/binnmus_needed.txt
237 238

# cleanup
239
rm $SOURCES $PACKAGES $TMPFILE
240
rm $TABLE_TODO $TABLE_DONE
241