bin/rblib/html.py

@@ -96,8 +96,9 @@ def create_main_navigation(suite=defaultsuite, arch=defaultarch,
                            displayed_page=None, suite_arch_nav_template=None,
                            ignore_experimental=False, no_suite=None,
                            no_arch=None):
-    suite_list, arch_list = _gen_suite_arch_nav_context(suite, arch,
-            suite_arch_nav_template, ignore_experimental, no_suite, no_arch)
+    suite_list, arch_list = _gen_suite_arch_nav_context(
+            suite, arch, suite_arch_nav_template,
+            ignore_experimental, no_suite, no_arch)
     context = {
         'suite': suite,
         'arch': arch,

bin/reproducible_alpine_manual_scheduler.sh

@@ -36,9 +36,9 @@ for PKG in $PACKAGES ; do
 	echo "Now trying to reschedule $PKG in $SUITE."
 	PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
 	if [ ! -z "${PKG_ID}" ] ; then
-		SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+		SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type = 'ci_build';")
 		if [ -z "$SCHEDULED" ] ; then
-			query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+			query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
 			SUCCESS="$SUCCESS $PKG"
 		else
 			echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."

bin/reproducible_alpine_scheduler.sh

@@ -47,7 +47,7 @@ update_alpine_repositories() {
 	done
 	TOTAL=$(cat ${ALPINE_PKGS}_full_pkgbase_list | wc -l)
 	echo "$(date -u ) - $TOTAL alpine packages are now known to alpine."
-	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	echo "$(date -u) - updating alpine repositories, currently $total packages scheduled."
 
 	#
@@ -109,7 +109,7 @@ update_alpine_repositories() {
 					echo "new package found: $REPO/$pkgbase $version "
 					query_db "INSERT into sources (name, version, suite, architecture, distribution) VALUES ('$PKG', '$version', '$SUITE', '$ARCH', $DISTROID);"
 					PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
-					query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+					query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
 				elif [ "$VERSION" != "$version" ] ; then
 					VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- /sbin/apk version -t $version $VERSION || true)"
 					if [ "$VERCMP" = ">" ] ; then
@@ -129,10 +129,10 @@ update_alpine_repositories() {
 							echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
 							PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
 							echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
-							SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+							SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type = 'ci_build';")
 							if [ -z "$SCHEDULED" ] ; then
-								echo " INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
-								query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+								echo " INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
+								query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
 							else
 								echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."
 							fi
@@ -170,12 +170,12 @@ update_alpine_repositories() {
 		AND s.architecture='x86_64'
 		AND (r.status LIKE 'DEPWAIT%' OR r.status LIKE '404%')
 		AND r.build_date < '$MINDATE'
-		AND s.id NOT IN (SELECT package_id FROM schedule)
+		AND s.id NOT IN (SELECT package_id FROM schedule WHERE build_type = 'ci_build')
 		LIMIT $MAX;"
 	local DEPWAIT404=$(query_db "$QUERY")
 	if [ ! -z "$DEPWAIT404" ] ; then
 		for PKG_ID in $DEPWAIT404 ; do
-			QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+			QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
 			query_db "$QUERY"
 		done
 		echo "$(date -u ) - done scheduling $(echo -n "$DEPWAIT404" | wc -l ) packages in DEPWAIT_ or 404_ states."
@@ -192,7 +192,7 @@ update_alpine_repositories() {
 	local THRESHOLD=600
 	MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "4 days ago")
 	SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
-	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	if [ $CURRENT -le $THRESHOLD ] ; then
 		echo "$(date -u ) - scheduling $MAX old packages."
 		QUERY="SELECT s.id, s.name, max(r.build_date) max_date
@@ -201,13 +201,13 @@ update_alpine_repositories() {
 			AND s.architecture='x86_64'
 			AND r.status != 'blacklisted'
 			AND r.build_date < '$MINDATE'
-			AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+			AND s.id NOT IN (SELECT schedule.package_id FROM schedule WHERE build_type = 'ci_build')
 			GROUP BY s.id, s.name
 			ORDER BY max_date
 			LIMIT $MAX;"
 		local OLD=$(query_db "$QUERY")
 		for PKG_ID in $(echo -n "$OLD" | cut -d '|' -f1) ; do
-			QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+			QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
 			query_db "$QUERY"
 		done
 		echo "$(date -u ) - done scheduling $MAX old packages."
@@ -219,7 +219,7 @@ update_alpine_repositories() {
 	# output stats
 	#
 	rm "$ALPINE_PKGS"_full_pkgbase_list
-	total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type = 'ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	new=$(cat $NEW | wc -l 2>/dev/null|| true)
 	updated=$(cat $UPDATED 2>/dev/null| wc -l || true)
 	old=$(echo -n "$OLD" | wc -l 2>/dev/null|| true)

bin/reproducible_archlinux_manual_scheduler.sh

@@ -35,9 +35,9 @@ for PKG in $PACKAGES ; do
 	echo "Now trying to reschedule $PKG in $SUITE."
 	PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
 	if [ ! -z "${PKG_ID}" ] ; then
-		SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+		SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type = 'ci_build';")
 		if [ -z "$SCHEDULED" ] ; then
-			query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+			query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
 			SUCCESS="$SUCCESS $PKG"
 		else
 			echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."

bin/reproducible_archlinux_scheduler.sh

@@ -47,7 +47,7 @@ update_archlinux_repositories() {
 		done | sort -u -R > "$ARCHLINUX_PKGS"_full_pkgbase_list
 	TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
 	echo "$(date -u ) - $TOTAL Arch Linux packages are now known to Arch Linux."
-	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE sch.build_type = 'ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	echo "$(date -u) - updating Arch Linux repositories, currently $total packages scheduled."
 
 	#
@@ -109,7 +109,7 @@ update_archlinux_repositories() {
 					echo "new package found: $REPO/$pkgbase $version "
 					query_db "INSERT into sources (name, version, suite, architecture, distribution) VALUES ('$PKG', '$version', '$SUITE', '$ARCH', $DISTROID);"
 					PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
-					query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+					query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
 				elif [ "$VERSION" != "$version" ] ; then
 					VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
 					if [ "$VERCMP" = "1" ] ; then
@@ -128,11 +128,11 @@ update_archlinux_repositories() {
 							echo $REPO/$pkgbase >> $UPDATED
 							echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
 							PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
-							echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
-							SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
+							echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID} AND build_type='ci_build';"
+							SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}' AND build_type='ci_build';")
 							if [ -z "$SCHEDULED" ] ; then
-								echo " INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
-								query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
+								echo " INSERT INTO schedule (package_id, date_scheduled build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
+								query_db "INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$DATE', 'ci_build');"
 							else
 								echo " $PKG (package_id: ${PKG_ID}) already scheduled, not scheduling again."
 							fi
@@ -170,12 +170,12 @@ update_archlinux_repositories() {
 		AND s.architecture='x86_64'
 		AND (r.status LIKE 'DEPWAIT%' OR r.status LIKE '404%')
 		AND r.build_date < '$MINDATE'
-		AND s.id NOT IN (SELECT package_id FROM schedule)
+		AND s.id NOT IN (SELECT package_id FROM schedule WHERE build_type='ci_build')
 		LIMIT $MAX;"
 	local DEPWAIT404=$(query_db "$QUERY")
 	if [ ! -z "$DEPWAIT404" ] ; then
 		for PKG_ID in $DEPWAIT404 ; do
-			QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+			QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) VALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
 			query_db "$QUERY"
 		done
 		echo "$(date -u ) - done scheduling $(echo -n "$DEPWAIT404" | wc -l ) packages in DEPWAIT_ or 404_ states."
@@ -192,7 +192,7 @@ update_archlinux_repositories() {
 	local THRESHOLD=600
 	MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "4 days ago")
 	SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
-	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	if [ $CURRENT -le $THRESHOLD ] ; then
 		echo "$(date -u ) - scheduling $MAX old packages."
 		QUERY="SELECT s.id, s.name, max(r.build_date) max_date
@@ -201,13 +201,13 @@ update_archlinux_repositories() {
 			AND s.architecture='x86_64'
 			AND r.status != 'blacklisted'
 			AND r.build_date < '$MINDATE'
-			AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+			AND s.id NOT IN (SELECT schedule.package_id FROM schedule WHERE build_type='ci_build')
 			GROUP BY s.id, s.name
 			ORDER BY max_date
 			LIMIT $MAX;"
 		local OLD=$(query_db "$QUERY")
 		for PKG_ID in $(echo -n "$OLD" | cut -d '|' -f1) ; do
-			QUERY="INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$SCHDATE');"
+			QUERY="INSERT INTO schedule (package_id, date_scheduled, build_type) 'ALUES ('${PKG_ID}', '$SCHDATE', 'ci_build');"
 			query_db "$QUERY"
 		done
 		echo "$(date -u ) - done scheduling $MAX old packages."
@@ -219,7 +219,7 @@ update_archlinux_repositories() {
 	# output stats
 	#
 	rm "$ARCHLINUX_PKGS"_full_pkgbase_list
-	total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
+	total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE build_type='ci_build' AND s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
 	new=$(cat $NEW | wc -l 2>/dev/null|| true)
 	updated=$(cat $UPDATED 2>/dev/null| wc -l || true)
 	old=$(echo -n "$OLD" | wc -l 2>/dev/null|| true)

bin/reproducible_build.sh

@@ -137,7 +137,7 @@ update_db_and_html() {
 	query_db "INSERT INTO results (package_id, version, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPKGID', '$VERSION', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB') ON CONFLICT (package_id) DO UPDATE SET version='$VERSION', status='$STATUS', build_date='$DATE', build_duration='$DURATION', node1='$NODE1', node2='$NODE2', job='$JOB' WHERE results.package_id='$SRCPKGID'"
 	query_db "INSERT INTO stats_build (name, version, suite, architecture, status, build_date, build_duration, node1, node2, job) VALUES ('$SRCPACKAGE', '$VERSION', '$SUITE', '$ARCH', '$STATUS', '$DATE', '$DURATION', '$NODE1', '$NODE2', '$JOB')"
 	# unmark build since it's properly finished
-	query_db "DELETE FROM schedule WHERE package_id='$SRCPKGID';"
+	query_db "DELETE FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build';"
 	gen_package_html $SRCPACKAGE
 	echo
 	echo "$(date -u) - successfully updated the database and updated $DEBIAN_URL/rb-pkg/${SUITE}/${ARCH}/$SRCPACKAGE.html"
@@ -413,6 +413,7 @@ choose_package() {
 		SELECT s.suite, s.id, s.name, s.version, sch.save_artifacts, sch.notify, s.notify_maintainer, sch.date_scheduled
 		FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
 		WHERE sch.date_build_started is NULL
+		AND sch.build_type='ci_build'
 		AND s.distribution=$DISTROID
 		AND s.architecture='$ARCH'
 		ORDER BY date_scheduled LIMIT 5"|sort -R|head -1)
@@ -442,12 +443,12 @@ choose_package() {
 	rm -f $BAD_BUILDS
 	# mark build attempt, first test if none else marked a build attempt recently
 	echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
-	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
+	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")
 	if [ -z "$RESULT" ] ; then
 		echo "ok, $SRCPACKAGE is not building anywhere…"
 		# try to update the schedule with our build attempt, then check no else did it, if so, abort
-		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
-		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
+		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND build_type='ci_build' AND date_build_started IS NULL"
+		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB' AND build_type='ci_build'")
 		if [ -z "$RESULT" ] ; then
 			echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
 			handle_race_condition

bin/reproducible_build_alpine_pkg.sh

@@ -53,7 +53,7 @@ update_pkg_in_db() {
         echo "$QUERY"
 	query_db "$QUERY"
         # unmark build since it's properly finished
-        QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}';"
+        QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}' AND build_type='ci_build';"
         echo "$QUERY"
 	query_db "$QUERY"
 	rm pkg.build_duration pkg.state pkg.version
@@ -68,6 +68,7 @@ choose_package() {
 		FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
 		WHERE distribution=$DISTROID
 		AND sch.date_build_started is NULL
+		AND sch.build_type='ci_build'
 		AND s.architecture='$ARCH'
 		ORDER BY date_scheduled LIMIT 10"|sort -R|head -1)
 	if [ -z "$RESULT" ] ; then
@@ -94,12 +95,12 @@ choose_package() {
 	rm -f $BAD_BUILDS
 	# mark build attempt, first test if none else marked a build attempt recently
 	echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
-	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
+	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")
 	if [ -z "$RESULT" ] ; then
 		echo "ok, $SRCPACKAGE is not building anywhere…"
 		# try to update the schedule with our build attempt, then check no else did it, if so, abort
-		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
-		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
+		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL AND build_type='ci_build'"
+		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB' AND build_type='ci_build'")
 		if [ -z "$RESULT" ] ; then
 			echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
 			handle_race_condition

bin/reproducible_build_archlinux_pkg.sh

@@ -53,7 +53,7 @@ update_pkg_in_db() {
         echo "$QUERY"
 	query_db "$QUERY"
         # unmark build since it's properly finished
-        QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}';"
+        QUERY="DELETE FROM schedule WHERE package_id='${SRCPKGID}' and build_type='ci_build';"
         echo "$QUERY"
 	query_db "$QUERY"
 	rm pkg.build_duration pkg.state pkg.version
@@ -68,6 +68,7 @@ choose_package() {
 		FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
 		WHERE distribution=$DISTROID
 		AND sch.date_build_started is NULL
+		AND sch.build_type='ci_build'
 		AND s.architecture='$ARCH'
 		ORDER BY date_scheduled LIMIT 10"|sort -R|head -1)
 	if [ -z "$RESULT" ] ; then
@@ -94,12 +95,12 @@ choose_package() {
 	rm -f $BAD_BUILDS
 	# mark build attempt, first test if none else marked a build attempt recently
 	echo "ok, let's check if $SRCPACKAGE is building anywhere yet…"
-	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID'")
+	RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")
 	if [ -z "$RESULT" ] ; then
 		echo "ok, $SRCPACKAGE is not building anywhere…"
 		# try to update the schedule with our build attempt, then check no else did it, if so, abort
-		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND date_build_started IS NULL"
-		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB'")
+		query_db "UPDATE schedule SET date_build_started='$DATE', job='$JOB' WHERE package_id='$SRCPKGID' AND build_type='ci_build' AND date_build_started IS NULL"
+		RESULT=$(query_db "SELECT date_build_started FROM schedule WHERE package_id='$SRCPKGID' AND date_build_started='$DATE' AND job='$JOB' AND build_type='ci_build'")
 		if [ -z "$RESULT" ] ; then
 			echo "hm, seems $SRCPACKAGE is building somewhere… failed to update the schedule table with our build ($SRCPKGID, $DATE, $JOB)."
 			handle_race_condition

bin/reproducible_common.sh

@@ -712,7 +712,7 @@ cleanup_pkg_files() {
 }
 
 handle_race_condition() {
-	local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID'")
+	local RESULT=$(query_db "SELECT job FROM schedule WHERE package_id='$SRCPKGID' AND build_type='ci_build'")  # XXX not only ci_build
 	local msg="Package ${SRCPACKAGE} (id=$SRCPKGID) in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n"
 	log_warning "$msg"
 	printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log
@@ -727,7 +727,7 @@ handle_race_condition() {
 unregister_build() {
 	# unregister this build so it will immeditiatly tried again
 	if [ -n "$SRCPKGID" ] ; then
-		query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id=$SRCPKGID"
+		query_db "UPDATE schedule SET date_build_started = NULL, job = NULL WHERE package_id=$SRCPKGID AND build_type='ci_build'"  # XXX not only ci_build
 	fi
 	NOTIFY=""
 }

bin/reproducible_db_maintenance.py

@@ -718,6 +718,14 @@ schema_updates = {
         "UPDATE stats_build SET status='reproducible' WHERE status='GOOD'",
         "UPDATE stats_build SET status='blacklisted' WHERE status='BLACKLISTED'"
     ],
+    46: [  # crete a build_type field
+        "CREATE TYPE build_type AS ENUM ('verification', 'ci_build')",
+        "ALTER TABLE results ADD COLUMN build_type build_type DEFAULT 'ci_build' NOT NULL",
+        "ALTER TABLE stats_build ADD COLUMN build_type build_type DEFAULT 'ci_build' NOT NULL",
+        "ALTER TABLE schedule ADD COLUMN build_type build_type",
+        "UPDATE schedule SET build_type='ci_build'",
+        "ALTER TABLE schedule ALTER COLUMN build_type SET NOT NULL",
+    ],
 }
 
 

bin/reproducible_remote_scheduler.py

@@ -217,7 +217,7 @@ def rest(scheduling_args, requester, local, suite, arch):
                 AND architecture='{arch}'"""
     query2 = """SELECT p.date_build_started
                 FROM sources AS s JOIN schedule as p ON p.package_id=s.id
-                WHERE p.package_id='{id}'"""
+                WHERE p.package_id='{id}' AND p.build_type='ci_build'"""
     for pkg in set(packages):
         # test whether the package actually exists
         result = query_db(query1.format(pkg=pkg, suite=suite, arch=arch))
@@ -316,6 +316,7 @@ def rest(scheduling_args, requester, local, suite, arch):
                 'save_artifacts': artifacts_value,
                 'notify': str(do_notify),
                 'scheduler': requester,
+                'build_type': 'ci_build',
             })
         else:
             add_to_schedule.append({
@@ -324,6 +325,7 @@ def rest(scheduling_args, requester, local, suite, arch):
                 'save_artifacts': artifacts_value,
                 'notify': str(do_notify),
                 'scheduler': requester,
+                'build_type': 'ci_build',
             })
 
         save_schedule.append({

bin/reproducible_scheduler.py

@@ -13,11 +13,9 @@
 import sys
 import lzma
 import deb822
-import smtplib
 import apt_pkg
 from sqlalchemy import sql
 from urllib.request import urlopen
-from email.mime.text import MIMEText
 from datetime import datetime, timedelta
 
 from rblib import query_db, db_table
@@ -449,7 +447,7 @@ def queue_packages(all_pkgs, packages, date):
 
 
 def schedule_packages(packages):
-    pkgs = [{'package_id': x, 'date_scheduled': packages[x]} for x in packages.keys()]
+    pkgs = [{'package_id': x, 'date_scheduled': packages[x], 'build_type': 'ci_build'} for x in packages.keys()]
     log.debug('IDs about to be scheduled: %s', packages.keys())
     if pkgs:
         conn_db.execute(db_table('schedule').insert(), pkgs)
@@ -469,7 +467,7 @@ def query_untested_packages(suite, arch, limit):
                     SELECT sources.id, sources.name FROM sources
                     WHERE sources.suite='{suite}' AND sources.architecture='{arch}'
                     AND sources.id NOT IN
-                       (SELECT schedule.package_id FROM schedule)
+                       (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
                     AND sources.id NOT IN
                        (SELECT results.package_id FROM results)
                     ORDER BY random()
@@ -488,7 +486,7 @@ def query_new_versions(suite, arch, limit):
                AND s.version != r.version
                AND r.status != 'blacklisted'
                AND s.id IN (SELECT package_id FROM results)
-               AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+               AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
                GROUP BY s.id, s.name, s.version, r.version
                ORDER BY max_date
                LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
@@ -513,7 +511,7 @@ def query_old_ftbfs_versions(suite, arch, limit):
                 AND r.status='FTBFS'
                 AND ( n.bugs = '[]' OR n.bugs IS NULL )
                 AND r.build_date < '{date}'
-                AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+                AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
                 GROUP BY s.id, s.name
                 ORDER BY max_date
                 LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -532,7 +530,7 @@ def query_old_depwait_versions(suite, arch, limit):
                 WHERE s.suite='{suite}' AND s.architecture='{arch}'
                 AND r.status='depwait'
                 AND r.build_date < '{date}'
-                AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+                AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
                 GROUP BY s.id, s.name
                 ORDER BY max_date
                 LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -552,7 +550,7 @@ def query_old_versions(suite, arch, limit):
                 WHERE s.suite='{suite}' AND s.architecture='{arch}'
                 AND r.status != 'blacklisted'
                 AND r.build_date < '{date}'
-                AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+                AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
                 GROUP BY s.id, s.name
                 ORDER BY max_date
                 LIMIT {limit}""".format(suite=suite, arch=arch,
@@ -570,7 +568,7 @@ def query_e404_versions(suite, arch, limit):
                 WHERE s.suite='{suite}' AND s.architecture='{arch}'
                 AND r.status = 'E404'
                 AND r.build_date < '{date}'
-                AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
+                AND s.id NOT IN (SELECT schedule.package_id FROM schedule AND build_type='ci_build')
                 GROUP BY s.id, s.name
                 ORDER BY max_date
                 LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit,
@@ -698,7 +696,7 @@ def schedule_e404_versions(arch, total):
 def scheduler(arch):
     query = "SELECT count(*) " + \
             "FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " + \
-            "WHERE s.architecture='{arch}'"
+            "WHERE s.architecture='{arch}' AND build_type='ci_build'"
     total = int(query_db(query.format(arch=arch))[0][0])
     log.info('==============================================================')
     log.info('Currently scheduled packages in all suites on ' + arch + ': ' + str(total))
@@ -734,7 +732,7 @@ def scheduler(arch):
     for suite in priotized_suite_order:
         query = "SELECT count(*) " \
                 "FROM schedule AS p JOIN sources AS s ON p.package_id=s.id " \
-                "WHERE s.suite='{suite}' AND s.architecture='{arch}'"
+                "WHERE s.suite='{suite}' AND s.architecture='{arch}' AND p.build_type='ci_build'"
         query = query.format(suite=suite, arch=arch)
         now_queued_here[suite] = int(query_db(query)[0][0]) + \
             len(untested[suite]+new[suite]+old[suite])
@@ -783,7 +781,7 @@ if __name__ == '__main__':
     purge_old_pages()
     query = "SELECT count(*) " + \
             "FROM schedule AS p JOIN sources AS s ON s.id=p.package_id " + \
-            "WHERE s.architecture='{}'"
+            "WHERE s.architecture='{}' AND build_type='ci_build'"
     message = ''
     for arch in ARCHS:
         log.info('Scheduling for %s...', arch)