TODO

@@ -330,7 +330,20 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
 ** create page for untested packages?
 
 * use db
-* then fix build.sh:
+** use debian tables with suite=archlinux_core etc and arch=x86_64 
+** problem: we currently have more (detailed) stati in archlinux
+** extend scheduler.sh to add packages to db and schedule them in db
+*** enable FIXME line once the db has been initially populated
+*** actually schedule old packages once queue is empty
+*** also delete unknown packages from db
+*** check/make sure that updated packages are only scheduled once
+** make build.sh read & write db
+** make build respect pacman exit code
+** create new job to recreate all pkg pages
+** call pkg html function after each build
+** cleanup now useless code from scheduler
+
+* fix build.sh:
 ** build2.log doesnt get deleted if build1 fails
 ** -> rename build2.log to $pkg_$version_build2.log
 

bin/reproducible_archlinux_scheduler.sh

@@ -14,14 +14,20 @@ set -e
 
 update_archlinux_repositories() {
 	echo "$(date -u) - Updating Arch Linux repositories, currently $(find $BASE/archlinux/ -name pkg.needs_build | wc -l ) packages scheduled."
+	#
+	# init
+	#
 	UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	OLDER=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 	local SESSION="archlinux-scheduler-$RANDOM"
 	schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
 	schroot --run-session -c $SESSION --directory /var/tmp -- sudo pacman -Syu --noconfirm
+
+	#
 	# Get a list of unique package bases.  Non-split packages don't have a pkgbase set
 	# so we need to use the pkgname for them instead.
+	#
 	schroot --run-session -c $SESSION --directory /var/tmp -- expac -S '%r %e %n %v' | \
 		while read repo pkgbase pkgname version; do
 			if [[ "$pkgbase" = "(null)" ]]; then
@@ -33,7 +39,9 @@ update_archlinux_repositories() {
 	TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
 	echo "$(date -u ) - $TOTAL Arch Linux packages are known in total to us."
 
+	#
 	# remove packages which are gone (only when run between 21:00 and 23:59)
+	#
 	if [ $(date +'%H') -gt 21 ] ; then
 		REMOVED=0
 		REMOVE_LIST=""
@@ -56,12 +64,49 @@ update_archlinux_repositories() {
 		fi
 	fi
 	
+	#
 	# schedule packages
+	#
 	for REPO in $ARCHLINUX_REPOS ; do
 		TMPPKGLIST=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
 		echo "$(date -u ) - updating list of available packages in repository '$REPO'."
 		grep "^$REPO" "$ARCHLINUX_PKGS"_full_pkgbase_list | \
 			while read repo pkgbase version; do
+				#
+				# db based scheduler
+				#
+				PKG=$pkgbase
+				SUITE="archlinux_$repo"
+				ARCH="x86_64"
+				VERSION=$(query_db "SELECT version FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+				DATE="$(date -u +'%Y-%m-%d %H:%M')"
+				if [ -z "$VERSION" ] ; then
+					# new package, add to db and schedule
+					query_db "INSERT into sources (name, version, suite, architecture) VALUES ('$PKG', '$VERSION', '$SUITE', '$ARCH');"
+					PKGID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+					#FIXME: enable next line once the db has been initially populated
+					# query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
+				elif [ "$VERSION" != "$version" ] ; then
+					VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
+					if [ "$VERCMP" = "1" ] ; then
+						# known package but with new version, update db and schedule
+						echo $REPO/$pkgbase >> $UPDATED
+						echo "$(date -u ) - we know $REPO/$pkgbase $VERSION, but repo has $version, so rescheduling... "
+						query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
+						PKGID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
+						query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('$PKGID', '$DATE');"
+
+					elif [ "$VERCMP" = "-1" ] ; then
+						# our version is higher than what's in the repo because we build trunk
+						echo "$REPO/$pkgbase $VERSION > $version" >> $OLDER
+					else
+						echo "$(date -u ) - This should never happen: we know about $pkgbase $VERSION, but repo has $version. \$VERCMP=$VERCMP"
+					fi
+				fi
+
+				#
+				# old scheduling to be deleted once the db is used
+				#
 				if [ ! -d $BASE/archlinux/$REPO/$pkgbase ] ; then
 					# schedule (all) entirely new packages
 					echo $REPO/$pkgbase >> $NEW
@@ -100,8 +145,12 @@ update_archlinux_repositories() {
 	schroot --end-session -c $SESSION
 	echo "$(date -u) - the following packages are known to us with higher versions than the repo because we build trunk:"
 	cat $OLDER
+	echo
+
+	#
 	# schedule up to $MAX packages we already know about
 	# (only if less than $THRESHOLD packages are currently scheduled)
+	#
 	old=""
 	local MAX=350
 	local THRESHOLD=450
@@ -124,6 +173,11 @@ update_archlinux_repositories() {
 		fi
 	fi
 	rm $OLDER
+	echo
+
+	#
+	# output stats
+	#
 	total=$(find $BASE/archlinux/ -name pkg.needs_build | wc -l )
 	rm "$ARCHLINUX_PKGS"_full_pkgbase_list
 	new=$(cat $NEW | wc -l 2>/dev/null|| true)

bin/reproducible_build.sh

@@ -20,28 +20,6 @@ common_init "$@"
 
 set -e
 
-log_info () {
-	_log "I:" "$@"
-}
-
-log_error () {
-	_log "E:" "$@"
-}
-
-log_warning () {
-	_log "W:" "$@"
-}
-
-log_file () {
-	cat $@ | tee -a $RBUILDLOG
-}
-
-_log () {
-	local prefix="$1"
-	shift 1
-	echo -e "$(date -u)  $prefix $*" | tee -a $RBUILDLOG
-}
-
 exit_early_if_debian_is_broken() {
 	# debian is fine, thanks
 	if false && [ "$ARCH" = "armhf" ] ; then
@@ -63,19 +41,6 @@ create_results_dirs() {
 	mkdir -vp $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}
 }
 
-handle_race_condition() {
-	local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
-	local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
-	log_warning "$msg"
-	printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
-	log_warning "Terminating this build quickly and nicely..."
-	if [ $SAVE_ARTIFACTS -eq 1 ] ; then
-		SAVE_ARTIFACTS=0
-		if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
-	fi
-	exit 0
-}
-
 save_artifacts() {
 		local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5)
 		local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_${ARCH}_tmp-${random}"

bin/reproducible_common.sh

@@ -114,6 +114,29 @@ if $DEBUG ; then
 	set -x
 fi
 
+# some cmomon logging functions
+log_info () {
+	_log "I:" "$@"
+}
+
+log_error () {
+	_log "E:" "$@"
+}
+
+log_warning () {
+	_log "W:" "$@"
+}
+
+log_file () {
+	cat $@ | tee -a $RBUILDLOG
+}
+
+_log () {
+	local prefix="$1"
+	shift 1
+	echo -e "$(date -u)  $prefix $*" | tee -a $RBUILDLOG
+}
+
 # sleep 1-23 secs to randomize start times
 delay_start() {
 	/bin/sleep $(echo "scale=1 ; $(shuf -i 1-230 -n 1)/10" | bc )
@@ -664,6 +687,19 @@ cleanup_pkg_files() {
 	rm -vf $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz}
 }
 
+handle_race_condition() {
+	local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
+	local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
+	log_warning "$msg"
+	printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
+	log_warning "Terminating this build quickly and nicely..."
+	if [ $SAVE_ARTIFACTS -eq 1 ] ; then
+		SAVE_ARTIFACTS=0
+		if [ ! -z "$NOTIFY" ] ; then NOTIFY="failure" ; fi
+	fi
+	exit 0
+}
+
 #
 # create the png (and query the db to populate a csv file...)
 #