Commit fa0996b4 authored by Holger Levsen's avatar Holger Levsen
Browse files

reproducible Archlinux: exclusivly use the db to track blacklisted packages 



Signed-off-by: default avatarHolger Levsen <holger@layer-acht.org>
parent 00bd6ac1

TODO

+0 −1
Original line number Diff line number Diff line
@@ -315,7 +315,6 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian 
** check for archlinux schroot sessions which should not be there and delete them. complain if that fails.



* use db - see https://tests.reproducible-builds.org/reproducibledb.htm

** blacklisting uses shell variables and db state, thats horrible

** scheduler.sh:

*** use asp to update trunk packages?

** html:

bin/reproducible_archlinux_scheduler.sh

+6 −7
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ update_archlinux_repositories() { 
	local UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)

	local NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)

	local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)

	local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"

	local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)

	echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."

	query_db "SELECT suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
@@ -115,12 +114,14 @@ update_archlinux_repositories() { 
					VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"

					if [ "$VERCMP" = "1" ] ; then

						# known package with new version, so update db and schedule

						echo $REPO/$pkgbase >> $UPDATED

						echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "

						query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"

						if [ -z $(echo $PKG | egrep -v "$BLACKLIST") ] ; then

						PKG_STATUS=$(query_db "SELECT status FROM results WHERE package_id='$PKG_ID';")

						if [ "$PKG_STATUS" = "BLACKLISTED" ] ; then

							echo "$PKG is blacklisted, so not scheduling it."

							continue

						else

							echo $REPO/$pkgbase >> $UPDATED

							echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "

							PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")

							echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"

							SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
@@ -188,12 +189,10 @@ update_archlinux_repositories() { 
	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")

	if [ $CURRENT -le $THRESHOLD ] ; then

		echo "$(date -u ) - scheduling $MAX old packages."

		# this query schedules blacklisted packages as they dont have status blacklisted in db...

		# FIXME: remove bash string blacklisting NOW

		QUERY="SELECT s.id, s.name, max(r.build_date) max_date

			FROM sources AS s JOIN results AS r ON s.id = r.package_id

			WHERE s.architecture='x86_64'

			AND r.status != 'blacklisted'

			AND r.status != 'BLACKLISTED'

			AND r.build_date < '$MINDATE'

			AND s.id NOT IN (SELECT schedule.package_id FROM schedule)

			GROUP BY s.id, s.name

bin/reproducible_common.sh

+13 −15
Original line number Diff line number Diff line
@@ -58,7 +58,6 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho 
# common settings for testing Arch Linux

ARCHLINUX_REPOS="core extra multilib community"

ARCHLINUX_PKGS=/srv/reproducible-results/archlinux_pkgs

ARCHLINUX_BLACKLISTED="syslinux ltrace gnutls haskell-hakyll mitmproxy pandoc python-pyftpdlib v2ray python-urllib3 perl-net-ssleay"

ARCHBASE=$BASE/archlinux



# common settings for testing rpm based distros
@@ -989,8 +988,6 @@ create_pkg_html() { 
	local buffer_message

	local STATE



	local blacklisted=false



	# clear files from previous builds

	cd "$ARCHLINUX_PKG_PATH"

	for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
@@ -1008,21 +1005,13 @@ create_pkg_html() { 
	#

	#

	if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then

		for i in $ARCHLINUX_BLACKLISTED ; do

			if [ "xxx${SRCPACKAGE}xxx" = "xxx${i}xxx" ] ; then

				blacklisted=true

			fi

		done

		# this horrible if elif elif elif elif...  monster is needed because

		# https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html

	        # has not yet been merged yet...

		# FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg



		# check different states and figure out what the page should look like

		if $blacklisted ; then

			STATE=BLACKLISTED

			buffer_message='blacklisted'

		elif find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then

		if find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then

			STATE=DEPWAIT_0

			buffer_message='could not resolve dependencies as there are conflicts'

		elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
@@ -1093,12 +1082,21 @@ create_pkg_html() { 
		elif find_in_buildlogs '==> ERROR: .* contains invalid characters:'; then

			STATE=FTBFS_4

			buffer_message='failed to build, pkg relations contain invalid characters'

		else

			STATE=query_db "SELECT r.status FROM results AS r

				JOIN sources as s on s.id=r.package_id

				WHERE s.architecture='x86_64'

				AND s.name='$SRCPACKAGE'

				AND s.suite='archlinux_$REPOSITORY';"

			if [ "$STATE" = "BLACKLISTED" ] ; then

				buffer_message='blacklisted'

			else

				STATE=UNKNOWN

				buffer_message='probably failed to build from source, please investigate'

			fi

		fi

		# print build failures

		if [[ $STATE = UNKNOWN ]]; then

		if [ "$STATE" = "UNKNOWN" ]; then

			echo "       $buffer_message" >> $HTML_BUFFER

		else

			include_icon $STATE "$buffer_message"