Commit fa0996b4 authored by Holger Levsen's avatar Holger Levsen

reproducible Archlinux: exclusivly use the db to track blacklisted packages

Signed-off-by: Holger Levsen's avatarHolger Levsen <holger@layer-acht.org>
parent 00bd6ac1
......@@ -315,7 +315,6 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian
** check for archlinux schroot sessions which should not be there and delete them. complain if that fails.
* use db - see https://tests.reproducible-builds.org/reproducibledb.htm
** blacklisting uses shell variables and db state, thats horrible
** scheduler.sh:
*** use asp to update trunk packages?
** html:
......
......@@ -23,7 +23,6 @@ update_archlinux_repositories() {
local UPDATED=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local NEW=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local BLACKLIST="/($(echo $ARCHLINUX_BLACKLISTED | sed "s# #|#g"))/"
local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)
echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."
query_db "SELECT suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
......@@ -115,12 +114,14 @@ update_archlinux_repositories() {
VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
if [ "$VERCMP" = "1" ] ; then
# known package with new version, so update db and schedule
echo $REPO/$pkgbase >> $UPDATED
echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
if [ -z $(echo $PKG | egrep -v "$BLACKLIST") ] ; then
PKG_STATUS=$(query_db "SELECT status FROM results WHERE package_id='$PKG_ID';")
if [ "$PKG_STATUS" = "BLACKLISTED" ] ; then
echo "$PKG is blacklisted, so not scheduling it."
continue
else
echo $REPO/$pkgbase >> $UPDATED
echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
......@@ -188,12 +189,10 @@ update_archlinux_repositories() {
local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
if [ $CURRENT -le $THRESHOLD ] ; then
echo "$(date -u ) - scheduling $MAX old packages."
# this query schedules blacklisted packages as they dont have status blacklisted in db...
# FIXME: remove bash string blacklisting NOW
QUERY="SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.architecture='x86_64'
AND r.status != 'blacklisted'
AND r.status != 'BLACKLISTED'
AND r.build_date < '$MINDATE'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
GROUP BY s.id, s.name
......
......@@ -58,7 +58,6 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho
# common settings for testing Arch Linux
ARCHLINUX_REPOS="core extra multilib community"
ARCHLINUX_PKGS=/srv/reproducible-results/archlinux_pkgs
ARCHLINUX_BLACKLISTED="syslinux ltrace gnutls haskell-hakyll mitmproxy pandoc python-pyftpdlib v2ray python-urllib3 perl-net-ssleay"
ARCHBASE=$BASE/archlinux
# common settings for testing rpm based distros
......@@ -989,8 +988,6 @@ create_pkg_html() {
local buffer_message
local STATE
local blacklisted=false
# clear files from previous builds
cd "$ARCHLINUX_PKG_PATH"
for file in build1.log build2.log build1.version build2.version *BUILDINFO.txt *.html; do
......@@ -1008,21 +1005,13 @@ create_pkg_html() {
#
#
if [ -z "$(cd $ARCHLINUX_PKG_PATH/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
for i in $ARCHLINUX_BLACKLISTED ; do
if [ "xxx${SRCPACKAGE}xxx" = "xxx${i}xxx" ] ; then
blacklisted=true
fi
done
# this horrible if elif elif elif elif... monster is needed because
# https://lists.archlinux.org/pipermail/pacman-dev/2017-September/022156.html
# has not yet been merged yet...
# FIXME: this has been merged, see http://jlk.fjfi.cvut.cz/arch/manpages/man/makepkg
# check different states and figure out what the page should look like
if $blacklisted ; then
STATE=BLACKLISTED
buffer_message='blacklisted'
elif find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
if find_in_buildlogs '^error: failed to prepare transaction \(conflicting dependencies\)'; then
STATE=DEPWAIT_0
buffer_message='could not resolve dependencies as there are conflicts'
elif find_in_buildlogs '==> ERROR: (Could not resolve all dependencies|.pacman. failed to install missing dependencies)'; then
......@@ -1094,11 +1083,20 @@ create_pkg_html() {
STATE=FTBFS_4
buffer_message='failed to build, pkg relations contain invalid characters'
else
STATE=UNKNOWN
buffer_message='probably failed to build from source, please investigate'
STATE=query_db "SELECT r.status FROM results AS r
JOIN sources as s on s.id=r.package_id
WHERE s.architecture='x86_64'
AND s.name='$SRCPACKAGE'
AND s.suite='archlinux_$REPOSITORY';"
if [ "$STATE" = "BLACKLISTED" ] ; then
buffer_message='blacklisted'
else
STATE=UNKNOWN
buffer_message='probably failed to build from source, please investigate'
fi
fi
# print build failures
if [[ $STATE = UNKNOWN ]]; then
if [ "$STATE" = "UNKNOWN" ]; then
echo " $buffer_message" >> $HTML_BUFFER
else
include_icon $STATE "$buffer_message"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment