Skip to content
Commits on Source (64)
Show whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 90b6fb21
......@@ -18,6 +18,6 @@ flake8:
image: debian:9
script:
- apt-get update
- apt-get install -y flake8
- flake8
- apt-get install -y python-flake8
- python -mflake8
allow_failure: true
config/common/variables
View file @ 90b6fb21
......@@ -82,4 +82,4 @@ declare -r LOCK_CHANGELOG="$lockdir/changelog.lock"
declare -r LOCK_STOP="${lockdir}/archive.stop"
# Lock for deploying new code
declare -r LOCK_DEPLOY="${lockdir}/deploy"
declare -r LOCK_DEPLOY=${LOCK_DEPLOY:-"${lockdir}/deploy"}
config/debian-security/cron.unchecked deleted 100755 → 0
View file @ a3532696
#! /bin/bash
set -e
set -o pipefail
set -u
export SCRIPTVARS=/srv/security-master.debian.org/dak/config/debian-security/vars
. $SCRIPTVARS
# And use one locale, no matter what the caller has set
export LANG=C
export LC_ALL=C
report=$queuedir/REPORT
reportdis=$queuedir/REPORT.disembargo
timestamp=$(date "+%Y-%m-%d %H:%M")
doanything=false
dopolicy=false
last_changed() {
psql -qAtc "SELECT MAX(last_changed) FROM suite WHERE archive_id=(SELECT id FROM archive WHERE name='$1')"
}
exec {LOCKFD}<>${LOCK_UNCHECKED}
flock --exclusive --timeout 600 ${LOCKFD}
old_last_changed=$(last_changed security)
old_last_changed_build=$(last_changed build-queues)
cd $unchecked
rsync -rtq --safe-links --chmod=F640,D755 --remove-source-files rsync-from-security-upload-master:/does/not/matter . || true
changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs)
if [ -n "$changes" ]; then
doanything=true
echo "$timestamp": ${changes:-"Nothing to do in unchecked"} >> $report
dak process-upload -a -d "$unchecked" >> $report
fi
cd $disembargo
changes=$(find . -maxdepth 1 -mindepth 1 -type f -name \*.changes | sed -e "s,./,," | xargs)
if [ -n "$changes" ]; then
doanything=true
echo "$timestamp": ${changes:-"Nothing to do in disembargo"} >> $reportdis
dak process-upload -a -d "$disembargo" >> $reportdis
dak manage-external-signature-requests
fi
for queue in embargoed unembargoed; do
echo "$timestamp: processing ${queue}" >> ${report}
dak process-policy ${queue} | mail -a "X-Debian: DAK" -e -s "Automatically accepted from ${queue}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" team@security.debian.org
done
filesaccepted=$(find ${queuedir}/accepted -type f -name "*.changes")
if [ -n "${filesaccepted}" ]; then
dopolicy=true
fi
# sync accepted files to ftpmaster
sync-accepted() {
cd ${base}
rsync --recursive --times --partial --remove-source-files \
${queuedir}/accepted/ rsync-to-ssh.upload.debian.org:/does/not/matter
}
if [ -n "${filesaccepted}" ]; then
sync-accepted || :
fi
# export policy queues
for queue in embargoed; do
cd ${queuedir}/${queue}
rm -rf export.new
mkdir export.new
dak export -q ${queue} -d export.new --all
rsync -a --delete export.new/. export/.
rm -rf export.new
cd ${base}
done
if [ "${doanything}" = "false" ] && [ "${dopolicy}" = "false" ]; then
echo "$timestamp": Nothing to do >> $report
exit 0
fi
# Update stable-kfreebsd
dak update-suite oldstable oldstable-kfreebsd
# manage build queues
dak manage-build-queues -a
# clean policy and build queues to remove files from rejected uploads
dak clean-suites -a new,policy,build-queues
# publish updates build queues
dak generate-packages-sources2 -a build-queues
dak generate-releases -a build-queues >/dev/null
${scriptsdir}/update-buildd-archive ${base}/build-queues ${incoming}/debian-security-buildd
$configdir/cron.buildd
new_last_changed=$(last_changed security)
new_last_changed_build=$(last_changed build-queues)
if [[ "${old_last_changed_build}" != "${new_last_changed_build}" ]]; then
sudo -u archvsync runmirrors -a security-buildd-pool
fi
if [[ "${old_last_changed}" != "${new_last_changed}" ]]; then
# We had something approved from a policy queue, push out new archive
exec {LOCKFD}>&-
$configdir/cronscript unchecked-dinstall
fi
trap - EXIT
config/debian-security/daily.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
# Note: Not explicitly unlocked, want it until cronscript ends, after
# all background functions finished
LOCK UNCHECKED 3600 exclusive false
......
config/debian-security/hourly.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
importusers none none false false
builddautosigning security none false false
config/debian-security/unchecked-dinstall.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
INCLUDE lock none true false
STATE startup none false false
LOCK DEPLOY 600 shared false
LOCK DAILY 3600 exclusive false
RMSTAGE none none false false
NOSTAGE startup none false false
......
config/debian-security/unchecked.functions 0 → 100644
View file @ 90b6fb21
function sync_accepted() {
rsync --recursive --times --partial --remove-source-files \
${queuedir}/accepted/ rsync-to-ssh.upload.debian.org:/does/not/matter
}
function update_kfreebsd() {
dak update-suite oldstable oldstable-kfreebsd
}
function trigger_wb_security() {
${configdir}/cron.buildd
}
function clean_suites() {
# clean policy and build queues to remove files from rejected uploads
dak clean-suites -a new,policy,build-queues
}
function export_policy() {
local queue=$1
cd ${queuedir}/${queue}
rm -rf export.new
mkdir export.new
dak export -q ${queue} -d export.new --all
rsync -a --delete export.new/. export/.
rm -rf export.new
}
function process_policy_security() {
local queue=$1
local report=$queuedir/REPORT
local timestamp=$(date "+%Y-%m-%d %H:%M")
echo "$timestamp: processing ${queue}" >> ${report}
dak process-policy ${queue} | mail -a "X-Debian: DAK" -e -s "Automatically accepted from ${queue}" -a "From: Debian FTP Masters <ftpmaster@ftp-master.debian.org>" team@security.debian.org
}
function do_dinstall() {
new_last_changed=$(last_changed security)
if [[ ${LAST_CHANGED} != ${new_last_changed} ]]; then
# We had something approved from a policy queue, push out new archive
includetasks unchecked-dinstall
fi
}
config/debian-security/unchecked.tasks 0 → 100644
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
LOCK UNCHECKED 600 exclusive false
pg_timestamp preunchecked none true false
fetchqueuedpackages rsync-from-security-upload-master fetchqueuedpackages false false
do_unchecked none unchecked false false
process_policy_security embargoed none false false
process_policy_security unembargoed none false false
export_policy embargoed none false false
sync_accepted none none false false
update_kfreebsd none none false false
make_buildd_dir security none false false
trigger_wb_security none none false false
clean_suites none none false false
UNLOCK UNCHECKED none false false
do_dinstall none dinstall false false
pg_timestamp postunchecked none false false
config/debian-security/unchecked.variables 0 → 100644
View file @ 90b6fb21
function last_changed() {
psql -qAtc "SELECT MAX(last_changed) FROM suite WHERE archive_id=(SELECT id FROM archive WHERE name='$1')"
}
declare -gr LAST_CHANGED=$(last_changed security)
config/debian-security/vars
View file @ 90b6fb21
......@@ -12,7 +12,7 @@ declare -r masterdir=${base}/dak/
export configdir=${configdir:-${masterdir}/config/${confpart}/}
# And the following types of cronscripts exists
declare -r POSSIBLEARGS='+(unchecked-dinstall|hourly|daily|weekly|mirror|deploy)'
declare -r POSSIBLEARGS='+(unchecked|unchecked-dinstall|hourly|daily|weekly|mirror|deploy)'
# And get all the tons of other variables
source ${configdir}/../common/variables
......
config/debian-security/weekly.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
LOCK UNCHECKED 1200 exclusive false
update_releases none none true false
INCLUDE mirror none true true
......
config/debian/common
View file @ 90b6fb21
......@@ -87,12 +87,19 @@ function log_timestamp() {
# used by cron.dinstall *and* cron.unchecked.
function make_buildd_dir () {
local archive=$1
# We generate straight into the static mirror location for incoming
log "Preparing buildd area"
dak manage-build-queues -a
dak generate-packages-sources2 -a build-queues
dak generate-releases -a build-queues >/dev/null
if [[ $archive = securiy ]]; then
${scriptsdir}/update-buildd-archive ${base}/build-queues ${incoming}/debian-security-buildd
$configdir/cron.buildd
sudo -u archvsync runmirrors -a security-buildd-pool
else
# Stick a last modified date in the page footer
echo "<p>Last updated: $(date -u)</p>" > ${incoming}/web/README.html
......@@ -103,6 +110,7 @@ function make_buildd_dir () {
sudo -u archvsync runmirrors -a buildd > ~dak/runmirrors-buildd.log 2>&1 &
chronic timeout -k 600 480 /usr/local/bin/static-update-component incoming.debian.org < /dev/null
wait
fi
}
# Process (oldstable)-proposed-updates "NEW" queue
......@@ -167,17 +175,17 @@ function do_unchecked () {
dak process-commands -d "$unchecked"
} >> ${report}
dak manage-external-signature-requests
sync_debbugs
make_buildd_dir
SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
ssh -q -q ${SSHOPT} wbadm@buildd /srv/wanna-build/trigger.often
else
log "Nothing to do"
echo "Nothing to do" >> ${report}
fi
}
function trigger_wb() {
SSHOPT="-n -o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
ssh -q -q ${SSHOPT} wbadm@buildd /srv/wanna-build/trigger.often
}
# process NEW policy queue
function do_new () {
log "Doing NEW processing"
......@@ -384,6 +392,14 @@ function linkmorgue() {
${scriptsdir}/link_morgue.sh
}
function fetchqueuedpackages() {
local host=$1
# Sync new uploaded packages from the central upload queue host.
log "Sync new uploads from upload queues"
cd ${unchecked}
rsync -rtq --safe-links --chmod=F640,D755 --remove-source-files $1:/does/not/matter . || true
}
########################################################################
########################################################################
########################################################################
......
config/debian/cronscript
View file @ 90b6fb21
......@@ -212,13 +212,20 @@ function lock() {
exit 21
fi
local LOCKFILE=
if [[ $LOCK == /* ]]; then
LOCKFILE=${LOCK}
else
# Prepend LOCK_ to lock name to get to variable name,
# kind of namespace
local lvar="LOCK_${LOCK}"
LOCKFILE=${!lvar}
fi
# Get filehandle
local randomstring
exec {randomstring}<>${!lvar}
exec {randomstring}<>${LOCKFILE}
# Store filehandle for later
LOCKFD[${LOCK}]=${randomstring}
......@@ -249,11 +256,11 @@ function lock() {
return
;;
3)
log_error "Could not get lock ${lvar}, timeout"
log_error "Could not get lock ${LOCKFILE}, timeout"
laststeps 2
;;
*)
log_error "Could not get lock ${lvar}"
log_error "Could not get lock ${LOCKFILE}"
laststeps 2
esac
}
......@@ -276,12 +283,6 @@ function laststeps() {
# and ensure its no longer used
exec > "$logdir/after${PROGRAM}.log" 2>&1
# If there is a postcronscript function, we run it, if script was successful
postfunc=$(type -t postcronscript || echo "")
if [[ ${successval} -eq 0 ]] && [[ -n ${postfunc} ]] && [[ ${postfunc} = function ]]; then
postcronscript
fi
# Now, at the very (successful) end of this run, make sure we remove
# our stage files, so the next script run will do it all again.
if [[ ${successval} -eq 0 ]]; then
......@@ -297,16 +298,6 @@ function laststeps() {
}
(
LOCKFREE=0
flock --shared --nonblock 42 || LOCKFREE=1
# Did we get the lock? (It's shared, so usually we will. But DSA
# can take an exclusive one in preparation for a reboot)
if [[ ${LOCKFREE} -gt 0 ]]; then
echo "Couldn't get (shared) reboot lock"
laststeps 1
fi
# Where we store lockfile filehandles
declare -A LOCKFD
......@@ -326,12 +317,6 @@ function laststeps() {
# Get rid of tempfiles at the end
trap cleanup EXIT TERM HUP INT QUIT
# If there is a precronscript function, we run it.
prefunc=$(type -t precronscript || echo "")
if [[ -n ${prefunc} ]] && [[ ${prefunc} = function ]]; then
precronscript
fi
# An easy access by name for the current log
ln -sf ${LOGFILE} ${logdir}/${PROGRAM}
......@@ -354,6 +339,4 @@ function laststeps() {
log "Cron script successful, all done"
laststeps 0
# And end the reboot-locked part
) 42</var/run/reboot-lock
)
config/debian/daily.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
wnppbugs none none false false
pushfilesdb none none false false
wbdump none none false false
......
config/debian/dinstall.functions
View file @ 90b6fb21
......@@ -644,6 +644,5 @@ function startup() {
function postcronscript() {
trap - ERR
logstats ${LOGFILE}
state "all done"
touch "${DINSTALLEND}"
}
config/debian/dinstall.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
INCLUDE lock none true false
STATE startup none false false
LOCK DEPLOY 600 shared false
LOCK DAILY 3600 exclusive false
NOSTAGE startup none false false
savetimestamp none none false false
......@@ -53,3 +53,5 @@ i18n2 none i18n\ 2 fal
stats none stats false true
testingsourcelist none none false true
cleantransactions none none false false
postcronscript none none false false
STATE all\ done none false false
config/debian/hourly.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
importusers none none false false
queuereport none none false true
removalstxt none none false true
......
config/debian/lock.tasks 0 → 100644
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK /run/reboot-lock 0 shared false
LOCK DEPLOY 600 shared false
config/debian/mirror.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
mirror none none true false
mirrorpush none none true false
config/debian/monthly.tasks
View file @ 90b6fb21
# FUNC ARGS TIME ERR BG
LOCK DEPLOY 600 shared false
INCLUDE lock none true false
rotatelog none none false true
rotatequeued none none false true
archivequeuedone none none false true
......