Skip to content

Commits on Source 2

Show whitespace changes
Inline Side-by-side
bin/reproducible_archlinux_manual_scheduler.sh
View file @ c891a865
......@@ -30,9 +30,10 @@ PACKAGES="$@"
SUCCESS=""
ALREADY_SCHEDULED=""
NOT_EXISTING=""
DISTROID=$(query_db "SELECT id FROM distributions WHERE name='archlinux'")
for PKG in $PACKAGES ; do
echo "Now trying to reschedule $PKG in $SUITE."
PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
if [ ! -z "${PKG_ID}" ] ; then
SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
if [ -z "$SCHEDULED" ] ; then
......
bin/reproducible_archlinux_scheduler.sh
View file @ c891a865
......@@ -25,7 +25,7 @@ update_archlinux_repositories() {
local KNOWN=$(mktemp -t archlinuxrb-scheduler-XXXXXXXX)
local TOTAL=$(cat ${ARCHLINUX_PKGS}_* | wc -l)
echo "$(date -u ) - $TOTAL Arch Linux packages were previously known to Arch Linux."
query_db "SELECT suite, name, version FROM sources WHERE architecture='$ARCH';" > $KNOWN
query_db "SELECT suite, name, version FROM sources WHERE distribution=$DISTROID AND architecture='$ARCH';" > $KNOWN
echo "$(date -u ) - $(cat $KNOWN | wc -l) Arch Linux packages are known in our database."
# init session
schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
......@@ -47,7 +47,7 @@ update_archlinux_repositories() {
done | sort -u -R > "$ARCHLINUX_PKGS"_full_pkgbase_list
TOTAL=$(cat ${ARCHLINUX_PKGS}_full_pkgbase_list | wc -l)
echo "$(date -u ) - $TOTAL Arch Linux packages are now known to Arch Linux."
local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
local total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
echo "$(date -u) - updating Arch Linux repositories, currently $total packages scheduled."
#
......@@ -70,7 +70,7 @@ update_archlinux_repositories() {
rm -r --one-file-system $BASE/archlinux/$REPO/$PKG
echo "$(date -u) - $REPO/$PKG removed as it's gone from the Archlinux repositories."
SUITE="archlinux_$repo"
PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
if [ -n "${PKG_ID}" ] ; then
query_db "DELETE FROM results WHERE package_id='${PKG_ID}';"
query_db "DELETE FROM schedule WHERE package_id='${PKG_ID}';"
......@@ -107,17 +107,19 @@ update_archlinux_repositories() {
# new package, add to db and schedule
echo $REPO/$pkgbase >> $NEW
echo "new package found: $repo/$pkgbase $version "
query_db "INSERT into sources (name, version, suite, architecture) VALUES ('$PKG', '$version', '$SUITE', '$ARCH');"
PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
DISTROID=$(query_db "SELECT id FROM distributions WHERE name = 'archlinux'")
query_db "INSERT into sources (name, version, suite, architecture, distribution) VALUES ('$PKG', '$version', '$SUITE', '$ARCH', $DISTROID);"
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
query_db "INSERT INTO schedule (package_id, date_scheduled) VALUES ('${PKG_ID}', '$DATE');"
elif [ "$VERSION" != "$version" ] ; then
VERCMP="$(schroot --run-session -c $SESSION --directory /var/tmp -- vercmp $version $VERSION || true)"
if [ "$VERCMP" = "1" ] ; then
# known package with new version, so update db and schedule
query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH';"
query_db "UPDATE sources SET version = '$version' WHERE name = '$PKG' AND suite = '$SUITE' AND architecture='$ARCH' AND distribution=$DISTROID;"
PKG_STATUS=$(query_db "SELECT r.status FROM results AS r
JOIN sources as s on s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.name='$PKG'
AND s.suite='$SUITE';")
if [ "$PKG_STATUS" = "BLACKLISTED" ] ; then
......@@ -126,7 +128,7 @@ update_archlinux_repositories() {
else
echo $REPO/$pkgbase >> $UPDATED
echo "$REPO/$pkgbase $VERSION is known in the database, but repo now has $version which is newer, so rescheduling... "
PKG_ID=$(query_db "SELECT id FROM sources WHERE name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
PKG_ID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$PKG' AND suite='$SUITE' AND architecture='$ARCH';")
echo " SELECT * FROM schedule WHERE package_id = '${PKG_ID}';"
SCHEDULED=$(query_db "SELECT * FROM schedule WHERE package_id = '${PKG_ID}';")
if [ -z "$SCHEDULED" ] ; then
......@@ -164,9 +166,10 @@ update_archlinux_repositories() {
local MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "24 hours ago")
local SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
QUERY="SELECT s.id FROM sources AS s
JOIN results as r on s.id=r.package_id
WHERE s.architecture='x86_64'\
AND (r.status LIKE 'DEPWAIT%' or r.status LIKE '404%')
JOIN results AS r ON s.id=r.package_id
WHERE s.distribution = $DISTROID
AND s.architecture='x86_64'
AND (r.status LIKE 'DEPWAIT%' OR r.status LIKE '404%')
AND r.build_date < '$MINDATE'
AND s.id NOT IN (SELECT package_id FROM schedule)
LIMIT $MAX;"
......@@ -190,12 +193,13 @@ update_archlinux_repositories() {
local THRESHOLD=450
MINDATE=$(date -u +"%Y-%m-%d %H:%M" -d "10 days ago")
SCHDATE=$(date -u +"%Y-%m-%d %H:%M" -d "7 days")
local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
local CURRENT=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
if [ $CURRENT -le $THRESHOLD ] ; then
echo "$(date -u ) - scheduling $MAX old packages."
QUERY="SELECT s.id, s.name, max(r.build_date) max_date
FROM sources AS s JOIN results AS r ON s.id = r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND r.status != 'BLACKLISTED'
AND r.build_date < '$MINDATE'
AND s.id NOT IN (SELECT schedule.package_id FROM schedule)
......@@ -216,7 +220,7 @@ update_archlinux_repositories() {
# output stats
#
rm "$ARCHLINUX_PKGS"_full_pkgbase_list
total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.architecture='x86_64' AND sch.date_build_started IS NULL;")
total=$(query_db "SELECT count(*) FROM sources AS s JOIN schedule AS sch ON s.id=sch.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND sch.date_build_started IS NULL;")
new=$(cat $NEW | wc -l 2>/dev/null|| true)
updated=$(cat $UPDATED 2>/dev/null| wc -l || true)
old=$(echo -n "$OLD" | wc -l 2>/dev/null|| true)
......
bin/reproducible_build_archlinux_pkg.sh
View file @ c891a865
......@@ -37,7 +37,7 @@ update_pkg_in_db() {
BUILD_STATE=$(cat pkg.state)
BUILD_VERSION="$(cat pkg.version)"
SUITE="archlinux_$REPOSITORY"
local SRCPKGID=$(query_db "SELECT id FROM sources WHERE name='$SRCPACKAGE' AND suite='$SUITE' AND architecture='$ARCH';")
local SRCPKGID=$(query_db "SELECT id FROM sources WHERE distribution=$DISTROID AND name='$SRCPACKAGE' AND suite='$SUITE' AND architecture='$ARCH';")
if [ -z "${SRCPKGID}" ] ; then
echo "${SRCPKGID} empty, ignoring $REPOSITORY/$SRCPACKAGE, failing hard."
exit 1
......@@ -48,8 +48,8 @@ update_pkg_in_db() {
DO UPDATE SET version='$BUILD_VERSION', status='$BUILD_STATE', build_date='$DATE', build_duration='$BUILD_DURATION', node1='$NODE1', node2='$NODE2', job='$BUILD_URL' WHERE results.package_id='$SRCPKGID'";
echo "$QUERY"
query_db "$QUERY"
QUERY="INSERT INTO stats_build (name, version, suite, architecture, status, build_date, build_duration, node1, node2, job)
VALUES ('$SRCPACKAGE', '$BUILD_VERSION', '$SUITE', '$ARCH', '$BUILD_STATE', '$DATE', '$BUILD_DURATION', '$NODE1', '$NODE2', '$BUILD_URL');"
QUERY="INSERT INTO stats_build (name, version, suite, architecture, distribution, status, build_date, build_duration, node1, node2, job)
VALUES ('$SRCPACKAGE', '$BUILD_VERSION', '$SUITE', '$ARCH', $DISTROID, '$BUILD_STATE', '$DATE', '$BUILD_DURATION', '$NODE1', '$NODE2', '$BUILD_URL');"
echo "$QUERY"
query_db "$QUERY"
# unmark build since it's properly finished
......@@ -66,7 +66,8 @@ choose_package() {
local RESULT=$(query_db "
SELECT s.suite, s.id, s.name, s.version
FROM schedule AS sch JOIN sources AS s ON sch.package_id=s.id
WHERE sch.date_build_started is NULL
WHERE distribution=$DISTROID
AND sch.date_build_started is NULL
AND s.architecture='$ARCH'
ORDER BY date_scheduled LIMIT 10"|sort -R|head -1)
if [ -z "$RESULT" ] ; then
......@@ -378,6 +379,7 @@ REPOSITORY=""
SRCPACKAGE=""
VERSION=""
SIZE=""
DISTROID=$(query_db "SELECT id FROM distributions WHERE name='archlinux'")
choose_package
mkdir -p $BASE/archlinux/$REPOSITORY/$SRCPACKAGE
# build package twice
......
bin/reproducible_db_maintenance.py
View file @ c891a865
......@@ -676,6 +676,14 @@ schema_updates = {
"INSERT INTO distributions (name) VALUES "
"('opensuse'), ('archlinux'), ('alpine')"
],
41: [ # mark archlinux packages as archlinx packages
"""UPDATE sources SET distribution=(
SELECT id FROM distributions WHERE name='archlinux')
WHERE suite LIKE 'archlinux_%%'""",
"""UPDATE stats_build SET distribution=(
SELECT id FROM distributions WHERE name='archlinux')
WHERE suite LIKE 'archlinux_%%'"""
],
}
......
bin/reproducible_html_archlinux.sh
View file @ c891a865
......@@ -54,16 +54,16 @@ repostats(){
echo "$(date -u) - starting to analyse build results for '$REPOSITORY'."
# prepare stats per repository
SUITE="archlinux_$REPOSITORY"
TOTAL=$(query_db "SELECT count(*) FROM sources AS s WHERE s.architecture='x86_64' AND s.suite='$SUITE';")
TESTED=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE';")
NR_GOOD=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status='GOOD';")
NR_FTBR=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'FTBR_%';")
NR_FTBFS=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'FTBFS_%';")
NR_DEPWAIT=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'DEPWAIT_%';")
NR_404=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE '404_%';")
NR_BLACKLISTED=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status='BLACKLISTED';")
NR_UNKNOWN=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'UNKNOWN_%';")
NR_UNTESTED=$(query_db "SELECT count(s.name) FROM sources AS s WHERE s.architecture='x86_64' AND s.suite='$SUITE' AND s.id NOT IN (SELECT package_id FROM results)")
TOTAL=$(query_db "SELECT count(*) FROM sources AS s WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE';")
TESTED=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE';")
NR_GOOD=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status='GOOD';")
NR_FTBR=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'FTBR_%';")
NR_FTBFS=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'FTBFS_%';")
NR_DEPWAIT=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'DEPWAIT_%';")
NR_404=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE '404_%';")
NR_BLACKLISTED=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status='BLACKLISTED';")
NR_UNKNOWN=$(query_db "SELECT count(*) FROM sources AS s JOIN results AS r ON s.id=r.package_id WHERE s.distribution=$DISTROID AND s.architecture='x86_64' AND s.suite='$SUITE' AND r.status LIKE 'UNKNOWN_%';")
NR_UNTESTED=$(query_db "SELECT count(s.name) FROM sources AS s WHERE s.architecture='x86_64' AND s.distribution=$DISTROID AND s.suite='$SUITE' AND s.id NOT IN (SELECT package_id FROM results)")
if [ $NR_UNTESTED -ne 0 ] ; then
let NR_UNKNOWN=$NR_UNKNOWN+$NR_UNTESTED
fi
......@@ -226,14 +226,16 @@ repository_pages(){
TESTED=$(query_db "SELECT count(*) FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE';")
write_page "<h2>$TESTED packages in repository $REPOSITORY</h2>"
include_pkg_table_header_in_page
REPO_PKGS=$(query_db "SELECT s.name FROM sources
AS s JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
ORDER BY r.status,s.name")
for SRCPACKAGE in $REPO_PKGS ; do
......@@ -253,12 +255,14 @@ state_pages(){
TESTED=$(query_db "SELECT count(*) FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND r.status LIKE '$STATE%';")
if [ "$STATE" = "UNKNOWN" ] ; then
# untested packages are also state UNKNOWN...
UNTESTED=$(query_db "SELECT count(s.name) FROM sources AS s
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.id NOT IN (SELECT package_id FROM results)")
if [ $UNTESTED -ne 0 ] ; then
let TESTED=$TESTED+$UNTESTED
......@@ -271,7 +275,8 @@ state_pages(){
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
AND r.status LIKE '$STATE%'
ORDER BY r.status,s.name")
......@@ -281,7 +286,8 @@ state_pages(){
if [ "$STATE" = "UNKNOWN" ] ; then
# untested packages are also state UNKNOWN...
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
AND s.id NOT IN (SELECT package_id FROM results)
ORDER BY s.name")
......@@ -306,13 +312,15 @@ repository_state_pages(){
TESTED=$(query_db "SELECT count(*) FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
AND r.status LIKE '$STATE%';")
if [ "$STATE" = "UNKNOWN" ] ; then
# untested packages are also state UNKNOWN...
UNTESTED=$(query_db "SELECT count(s.name) FROM sources AS s
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
AND s.id NOT IN (SELECT package_id FROM results)")
if [ $UNTESTED -ne 0 ] ; then
......@@ -324,7 +332,8 @@ repository_state_pages(){
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
AND r.status LIKE '$STATE%'
ORDER BY r.status,s.name")
......@@ -334,7 +343,8 @@ repository_state_pages(){
if [ "$STATE" = "UNKNOWN" ] ; then
# untested packages are also state UNKNOWN...
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND s.suite='$SUITE'
AND s.id NOT IN (SELECT package_id FROM results)
ORDER BY s.name")
......@@ -357,14 +367,16 @@ recent_builds_page(){
RECENT=$(query_db "SELECT count(s.name) FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND r.build_date > '$MAXDATE'")
write_page "<h2>$RECENT builds of Archlinux packages in the last 24h</h2>"
include_pkg_table_header_in_page
STATE_PKGS=$(query_db "SELECT s.name FROM sources AS s
JOIN results AS r
ON s.id=r.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND r.build_date > '$MAXDATE'
ORDER BY r.build_date
DESC")
......@@ -384,7 +396,8 @@ currently_scheduled_page(){
FROM sources AS s
JOIN schedule AS sch
ON s.id=sch.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND sch.date_build_started IS NULL")
write_page "<h2>Currently $TESTED scheduled builds of Archlinux packages</h2>"
write_page " <table><tr><th>source package</th><th>repository</th><th>version</th><th>scheduled</th></tr>"
......@@ -392,7 +405,8 @@ currently_scheduled_page(){
FROM sources AS s
JOIN schedule AS sch
ON s.id=sch.package_id
WHERE s.architecture='x86_64'
WHERE s.distribution=$DISTROID
AND s.architecture='x86_64'
AND sch.date_build_started IS NULL
ORDER BY sch.date_scheduled, s.name")
OIFS=$IFS
......@@ -419,6 +433,7 @@ TITLE=""
STATE=""
REPOSITORY=""
SRCPACKAGE=""
DISTROID=$(query_db "SELECT id FROM distributions WHERE name='archlinux'")
if [ -z "$1" ] ; then
MEMBERS_FTBFS="0 1 2 3 4"
......