reproducible_html_repository_comparison.sh 8.81 KB
Newer Older
1
#!/bin/bash
2
# vim: set noexpandtab:
3

4
# Copyright 2015-2018 Holger Levsen <holger@layer-acht.org>
5 6
# released under the GPLv=2

7
DEBUG=false
8 9 10 11 12
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
13

14
VIEW=repositories
15
PAGE=index_${VIEW}.html
Holger Levsen's avatar
Holger Levsen committed
16
SOURCES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
17
PACKAGES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
18
TMPFILE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
19 20
TABLE_TODO=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
TABLE_DONE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX)
21

22
MODIFIED_IN_SID=0
23
MODIFIED_IN_EXP=0
24
BINNMUS_NEEDED=0
25

26 27 28 29
write_row() {
        echo "$1" >> $ROW
}

30
custom_curl() {
31
	echo -n "$(date -u) - downloading $1 to $2 - "
32
	curl -s $1 > $2
33
	local SIZE=$(ls -la $2 |cut -d " " -f5)
34
	echo "$SIZE bytes."
35 36
}

37
ARCH="amd64"
38
SUITE="unstable"
Holger Levsen's avatar
Holger Levsen committed
39
echo "$(date -u) - starting to write $PAGE page. Downloading Sources and Packages files from our repository."
Holger Levsen's avatar
Holger Levsen committed
40
write_page_header $VIEW "Comparison between the reproducible builds apt repository and regular Debian suites"
41 42 43
write_page "<p>These source packages (and their binaries packages) are different from unstable in our experimental apt repository. They are available for <a href=\"https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain#Usage_example\">testing using these sources.lists</a> entries:<pre>"
write_page "deb https://tests.reproducible-builds.org/debian/repository/debian/ ./"
write_page "deb-src https://tests.reproducible-builds.org/debian/repository/debian/ ./"
44
write_page "</pre></p>"
45

46 47
custom_curl https://tests.reproducible-builds.org/debian/repository/debian/Sources $SOURCES
custom_curl https://tests.reproducible-builds.org/debian/repository/debian/Packages $PACKAGES
Holger Levsen's avatar
Holger Levsen committed
48
SOURCEPKGS=$(grep-dctrl -n -s Package -r -FPackage . $SOURCES | sort -u)
49 50
echo

Holger Levsen's avatar
Holger Levsen committed
51
for PKG in $SOURCEPKGS ; do
52
	echo "$(date -u) - Processing $PKG..."
53 54 55 56 57
	if [ "${PKG:0:3}" = "lib" ] ; then
		PREFIX=${PKG:0:4}
	else
		PREFIX=${PKG:0:1}
	fi
Holger Levsen's avatar
Holger Levsen committed
58
	VERSIONS=$(grep-dctrl -n -s version -S $PKG $SOURCES|sort -u)
59
	CRUFT=""
60
	BET=""
61
	OBSOLETE_IN_SID=false
62
	OBSOLETE_IN_BUSTER=false
63
	OBSOLETE_IN_EXP=false
64 65 66
	#
	# gather versions of a package
	#
67 68 69 70 71 72 73 74
	for VERSION in ${VERSIONS} ; do
		if [ "$BET" = "" ] ; then
			BET=${VERSION}
			continue
		elif dpkg --compare-versions "$BET" lt "${VERSION}"  ; then
			BET=${VERSION}
		fi
	done
75
	SID=$(rmadison -s unstable $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo)
76 77
	for VERSION in ${VERSIONS} ; do
		if [ "${VERSION}" != "$BET" ] ; then
Holger Levsen's avatar
Holger Levsen committed
78
			CRUFT="$CRUFT ${VERSION}"
79 80
		fi
	done
81
	BUSTER=$(rmadison -s buster $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo)
82
	EXPERIMENTAL=$(rmadison -s experimental $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo)
83
	#
84
	# format output
85 86
	#
	CSID=""
87
	CBUSTER=""
88
	CEXP=""
89 90
	if [ ! -z "$BUSTER" ] ; then
		for i in $BUSTER ; do
91
			if dpkg --compare-versions "$i" gt "$BET" ; then
92 93
				CBUSTER="$CBUSTER<span class=\"green\">$i</span><br />"
				OBSOLETE_IN_BUSTER=true
94 95
				OBSOLETE_IN_SID=true
				OBSOLETE_IN_EXP=true
96
			else
97
				CBUSTER="$CBUSTER$i<br />"
98
			fi
99 100
		done
	fi
Holger Levsen's avatar
Holger Levsen committed
101 102
	if [ ! -z "$EXPERIMENTAL" ] ; then
		for i in $EXPERIMENTAL ; do
103 104
			if dpkg --compare-versions "$i" gt "$BET" ; then
				CEXP="$CEXP<a href=\"https://tracker.debian.org/media/packages/$PREFIX/$PKG/changelog-$i\">$i</a><br />"
105 106
				OBSOLETE_IN_EXP=true
		else
107 108
				CEXP="$CEXP$i<br />"
			fi
109
		done
110
	fi
111 112 113 114 115 116 117
	for i in $SID ; do
		if dpkg --compare-versions "$i" gt "$BET" ; then
			CSID="$CSID<a href=\"https://tracker.debian.org/media/packages/$PREFIX/$PKG/changelog-$i\">$i</a><br />"
			if [ ! -z "$BET" ] ; then
				CRUFT="$BET $CRUFT"
				BET=""
				OBSOLETE_IN_SID=true
118
				OBSOLETE_IN_EXP=true
119 120 121 122 123
			fi
		else
			CSID="$CSID$i<br />"
		fi
	done
124
	CBINARIES=""
125
	if [ ! -z "$BET" ] ; then
126 127
		ONLYALL=true
		for ARCH in all ${ARCHS} ; do
128
			i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)"
129 130 131
			if [ "$ARCH" != "all" ] && [ ! -z "$i" ] ; then
				ONLYALL=false
			fi
132
			echo " $ARCH: $i"
133 134
		done
		for ARCH in all ${ARCHS} ; do
135
			i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)"
136
			if [ ! -z "$i" ] ; then
137 138
				i="$ARCH: $i"
			elif [ -z "$i" ] && [ "$ARCH" != "all" ] && ! $ONLYALL ; then
139
				i="<span class=\"red\">no binaries for $ARCH</span>"
140
				let "BINNMUS_NEEDED+=1"
141
			fi
142
			CBINARIES="$CBINARIES<br />$i"
143
		done
144
		BET="<span class=\"green\">$BET</span>"
145
		ROW=$TABLE_TODO
146 147
	else
		BET="&nbsp;"
148
		ROW=$TABLE_DONE
149 150 151 152
	fi
	if [ ! -z "$CRUFT" ] ; then
		CRUFT="$(echo $CRUFT|sed 's# #<br />#g')"
	fi
153 154 155
	#
	# write output
	#
156 157
	write_row "<tr><td><pre>src:$PKG</pre></td>"
	write_row " <td>"
158
	GIT="$PKG.git"
159
	URL="https://salsa.debian.org/reproducible-builds/packages/$PKG/tree/pu/reproducible_builds"
160
	custom_curl $URL $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
161
	if [ "$(grep "'error'>No repositories found" $TMPFILE 2>/dev/null)" ] ; then
162
		write_row "<span class=\"red\">no git repository found:</span><br />$URL"
Holger Levsen's avatar
Holger Levsen committed
163
	elif [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then
164
		URL="https://salsa.debian.org/reproducible-builds/packages/$PKG/tree/merged/reproducible_builds"
165
		custom_curl $URL $TMPFILE
Holger Levsen's avatar
Holger Levsen committed
166
		if [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then
167
			if ! $OBSOLETE_IN_SID ; then
168
				write_row "<a href=\"$URL\">$GIT</a><br /><span class=\"purple\">non-standard branch</span>"
169
			else
170
				write_row "<a href=\"$URL\">$GIT</a><br /><span class=\"green\">non-standard branch</span> (but that is ok, our package ain't used in unstable)"
171 172
			fi
		else
173
			write_row "<a href=\"$URL\">$GIT</a>"
Holger Levsen's avatar
Holger Levsen committed
174
			MERGEINFO=""
175 176
			if $OBSOLETE_IN_BUSTER ; then
				MERGEINFO=" and available in buster and unstable"
177
			elif $OBSOLETE_IN_SID ; then
178
				MERGEINFO=" and available in unstable"
179
			elif $OBSOLETE_IN_EXP ; then
180
				MERGEINFO=" and available in experimental"
181
			fi
182
			write_row "<br />(<span class=\"green\">merged</span>$MERGEINFO)"
183
		fi
184
	else
185
		write_row "<a href=\"$URL\">$GIT</a>"
186 187 188 189 190 191 192 193
		if $OBSOLETE_IN_BUSTER && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
			write_row "<br />(unused?"
			write_row "<br /><span class=\"purple\">Then the branch should probably renamed.</span>)"
		elif $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
			write_row "<br />(only used in buster, fixed in sid,"
			write_row "<br /><span class=\"purple\">branch probably either should be renamed to <em>merged/reproducible_builds</em> or a new upload to our repo is needed?</span>)"
		elif $OBSOLETE_IN_EXP ; then
			write_row "<br />(only used in buster and unstable, fixed in experimental)"
194
		fi
195
	fi
196 197 198
	if ! $OBSOLETE_IN_SID ; then
		let "MODIFIED_IN_SID+=1"
	fi
199 200 201
	if ! $OBSOLETE_IN_EXP ; then
		let "MODIFIED_IN_EXP+=1"
	fi
202 203
	write_row " </td>"
	write_row " <td><a href=\"https://tracker.debian.org/pkg/$PKG\">PTS</a></td>"
204 205 206
	URL="https://bugs.debian.org/cgi-bin/pkgreport.cgi?src=$PKG&users=reproducible-builds@lists.alioth.debian.org&archive=both"
	for TAG in $USERTAGS ; do
		URL="$URL&tag=$TAG"
207
	done
208 209 210
	write_row " <td><a href=\"$URL\">bugs</a></td>"
	write_row " <td>$CRUFT</td>"
	write_row " <td>$BET $CBINARIES</td>"
211
	write_row " <td>$CBUSTER</td>"
212 213 214
	write_row " <td>$CSID</td>"
	write_row " <td>$CEXP</td>"
	write_row "</tr>"
215
	echo "$(date -u) - Package $PKG done."
216
	echo
217
done
218
if [ -s $TABLE_TODO ] ; then
219
	write_page "<p><table><tr><th class=\"center\">package</th><th class=\"center\">git repo</th><th class=\"center\">PTS link</th><th class=\"center\">usertagged bug(s)</th><th class=\"center\">old versions in our repo</th><th class=\"center\">version in our repo<br />(available binary packages per architecture)</th><th class=\"center\">version in 'buster'</th><th class=\"center\">version in 'unstable'</th><th class=\"center\">version in 'experimental'</th></tr>"
220 221 222 223 224
	cat $TABLE_TODO >> $PAGE
	write_page "</table></p>"
else
	write_page "<p>Congratulations! There are no modified packages in our repository compared to unstable. (Yes, that means our repository is obsolete now.)"
fi
225
if [ -s $TABLE_DONE ] ; then
226
	write_page "<p><table><tr><th class=\"center\">obsoleted package,<br />version in sid higher than in our repo</th><th class=\"center\">git repo</th><th class=\"center\">PTS link</th><th class=\"center\">usertagged bug(s)</th><th class=\"center\">old version(s) in our repo</th><th class=\"center\">version in our repo<br />(available binary packages per architecture)</th><th class=\"center\">version in 'buster'</th><th class=\"center\">version in 'unstable'</th><th class=\"center\">version in 'experimental'</th></tr>"
227 228 229
	cat $TABLE_DONE >> $PAGE
	write_page "</table></p>"
fi
230
write_page_footer
231
publish_page debian
232
echo "$MODIFIED_IN_SID" > /srv/reproducible-results/modified_in_sid.txt
233
echo "$MODIFIED_IN_EXP" > /srv/reproducible-results/modified_in_exp.txt
234
echo "$BINNMUS_NEEDED" > /srv/reproducible-results/binnmus_needed.txt
235 236

# cleanup
237
rm $SOURCES $PACKAGES $TMPFILE
238
rm $TABLE_TODO $TABLE_DONE