cronscript 10.9 KB
Newer Older
1 2 3
#!/bin/bash
# No way I try to deal with a crippled sh just for POSIX foo.

4
# Copyright (C) 2009-2016, 2018 Joerg Jaspert <joerg@debian.org>
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

# Homer: Are you saying you're never going to eat any animal again? What
#        about bacon?
# Lisa: No.
# Homer: Ham?
# Lisa: No.
# Homer: Pork chops?
# Lisa: Dad, those all come from the same animal.
# Homer: Heh heh heh. Ooh, yeah, right, Lisa. A wonderful, magical animal.

# exit on errors
set -e
Joerg Jaspert's avatar
Joerg Jaspert committed
30 31 32
# A pipeline's return status is the value of the last (rightmost)
# command to exit with a non-zero status, or zero if all commands exit
# successfully.
33 34 35 36 37 38 39 40
set -o pipefail
# make sure to only use defined variables
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the functions is
# the important part here)
set -E

41 42 43
# If the extglob shell option is enabled using the shopt builtin,
# several extended pattern matching operators are recognized. We use
# it for the POSSIBLEARGS and the first case ${ARGS} matching.
Joerg Jaspert's avatar
Joerg Jaspert committed
44 45
shopt -s extglob

46
# And use one locale, no matter what the caller has set
Joerg Jaspert's avatar
Joerg Jaspert committed
47 48
export LANG=C.UTF-8
export LC_ALL=C.UTF-8
49

Joerg Jaspert's avatar
Joerg Jaspert committed
50 51
# One arg please
declare -lr ARG=${1:-"meh"}
52

Joerg Jaspert's avatar
Joerg Jaspert committed
53 54 55
# program name is the (lower cased) first argument.
PROGRAM="${ARG}"

56 57 58 59 60
# import the general variable set. (This will overwrite configdir, but
# it is expected to have the same value)
export SCRIPTVARS=${configdir:?Please define configdir to run this script}/vars
. ${SCRIPTVARS}

61 62 63
# set DEBUG if you want to see a little more logs (needs to be used more)
DEBUG=${DEBUG:-0}

Joerg Jaspert's avatar
Joerg Jaspert committed
64 65 66 67 68 69 70
# Check if the argument is a known one. If so, lock us so that only
# one copy of the type of cronscript runs. The $type.tasks file is
# mandantory, so use that for locking.
case ${ARG} in
    ${POSSIBLEARGS})
        # Only one of me should ever run.
        FLOCKER=${FLOCKER:-""}
71
        [[ ${FLOCKER} != ${configdir}/${PROGRAM}.tasks ]] && exec env FLOCKER="${configdir}/${PROGRAM}.tasks" flock -E 0 -en "${configdir}/${PROGRAM}.tasks" "$0" "$@" || :
Joerg Jaspert's avatar
Joerg Jaspert committed
72 73 74
        ;;
    *)
        cat - <<EOF
75 76
This is the cronscript. It needs an argument or it won't do anything
for you.
77

78
Currently accepted Arguments: ${POSSIBLEARGS}
79

80 81
To see what they do, you want to look at the files
\$ARGUMENT.{tasks,functions,variables} in ${configdir}.
82 83

EOF
Joerg Jaspert's avatar
Joerg Jaspert committed
84 85 86
        exit 0
        ;;
esac
87

88 89 90 91 92 93 94
function includetasks() {
    local NAME=${1:?}

    _preparetasks ${NAME}
    _runtasks ${NAME}
}

95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
function _preparetasks() {
    local NAME=${1:?}

    # Each "cronscript" may have a variables and a functions file
    # that we source
    for what in variables functions; do
        if [[ -f ${configdir}/${NAME}.${what} ]]; then
            . ${configdir}/${NAME}.${what}
        fi
    done
}

function _runtasks() {
    local NAME=${1:?}

    # Which list of tasks should we run?
    local TASKLIST="${configdir}/${NAME}.tasks"

    # This loop simply wants to be fed by a list of values (see below)
    # made out of 5 columns.
    # The first four are the array values for the stage function, the
    # fifth tells us if we should background the stage call.
    #
    #  - FUNC - the function name to call
    #  - ARGS - Possible arguments to hand to the function. Can be the empty string
    #  - TIME - The timestamp name. Can be the empty string
    #  - ERR  - if this is the string false, then the call will be surrounded by
    #           set +e ... set -e calls, so errors in the function do not exit
    #           the script. Can be the empty string, meaning true.
    #  - BG   - Background the function stage?
    #
    # ATTENTION: Spaces in arguments or timestamp names need to be escaped by \
    #
    # NOTE 1: There are special values for the first column (FUNC).
    #         NOSTAGE - do not call stage function, call the command directly.
    #         RMSTAGE - clean out the stages directory, and as such
    #                   the recording what already ran in an earlier cronscript.
    #                   Note: Only really makes sense at beginning of a tasks file,
    #                   the stages directory gets cleared at successful exit anyways.
    #                   RMSTAGE simply ensures that ALL of the crons tasks ALWAYS run.
135
    #         INCLUDE - Runs another task list after including corresponding functions
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179

    # Note 2: If you want to hand an empty value to the stage function,
    #         use the word "none" in the list below.
    while read FUNC ARGS TIME ERR BACKGROUND; do
        debug "FUNC: $FUNC ARGS: $ARGS TIME: $TIME ERR: $ERR BG: $BACKGROUND"

        # Empty values in the value list are the string "none" (or the
        # while read loop won't work). Here we ensure that variables that
        # can be empty, are empty if the string none is set for them.
        for var in ARGS TIME; do
            if [[ ${!var} == none ]]; then
                typeset ${var}=''
            fi
        done

        # ERR/BACKGROUND are boolean for all but LOCK/UNLOCK, check that they are.
        for var in ERR BACKGROUND; do
            if [[ ${!var} != false ]] && [[ ${!var} != true ]]; then
                if [[ ${FUNC} != LOCK ]] && [[ ${FUNC} != UNLOCK ]]; then
                    error "Illegal value ${!var} for ${var} (should be true or false), line for function ${FUNC}"
                fi
            fi
        done
        case ${FUNC} in
            NOSTAGE)
                ${ARGS}
                ;;
            RMSTAGE)
                # Make sure we remove our stage files, so all the
                # actions will be done again.
                rm -f ${stagedir}/*
                ;;
            LOCK)
                # We are asked to set a lock, so try to get it.
                # For this we redefine what the columns mean.
                # ARGS: Name of the lockfile
                # TIME: How long to wait for getting the (exclusive) lock
                # ERR: shared == shared lock, may be hold more than once, exclusive == exclusive, only one.

                lock ${ARGS} ${TIME} ${ERR}
                ;;
            UNLOCK)
                unlock ${ARGS}
                ;;
180 181 182
            INCLUDE)
                includetasks ${ARGS}
                ;;
183 184 185 186 187 188 189 190 191 192 193 194 195
            *)
                GO=(
                    FUNC=${FUNC}
                    TIME=${TIME}
                    ARGS=${ARGS}
                    ERR=${ERR}
                )
                if [[ ${BACKGROUND} == true ]]; then
                    stage $GO &
                else
                    stage $GO
                fi
                ;;
196
        esac < /dev/null
197 198 199
    done < <(grep -v '^#' ${TASKLIST} )
}

200 201 202 203 204 205 206 207 208 209
function lock() {
    local LOCK=${1:-}
    local TIME=${2:-600}
    local TYPE=${3:-exclusive}

    if [[ -z ${LOCK} ]]; then
        log_error "No lockfile name given"
        exit 21
    fi

210 211 212 213 214 215 216 217 218 219
    local LOCKFILE=

    if [[ $LOCK == /* ]]; then
        LOCKFILE=${LOCK}
    else
        # Prepend LOCK_ to lock name to get to variable name,
        # kind of namespace
        local lvar="LOCK_${LOCK}"
        LOCKFILE=${!lvar}
    fi
220

221 222 223 224 225 226 227 228 229
    # bash can't open a file read-only, while creating it,
    # so we need to create it ourselves.
    if ! [[ -e $LOCKFILE ]]; then
        install -m 444 /dev/null $LOCKFILE || {
            log_error "Could not create lock ${LOCKFILE}"
            laststeps 2
        }
    fi

230 231
    # Get filehandle
    local randomstring
232
    exec {randomstring}<${LOCKFILE}
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
    # Store filehandle for later
    LOCKFD[${LOCK}]=${randomstring}

    # "Abusing" the err column, expecting the shared/exclusive value there.
    # Any wrong value means exclusive.
    case ${ERR} in
        shared|exclusive)
            flockparm="--${ERR}"
            ;;
        *)
            flockparm="--exclusive"
            ;;
    esac

    # Deal with time being special, usually it means false or true,
    # but for locks we want a timeout. So if its set to one of the usuals,
    # assume 300
    if [[ ${TIME} == none ]]; then
        TIME=300
    fi
    # Now try to get the lock
254 255 256 257 258 259 260 261 262
    set +e
    flock ${flockparm} --timeout ${TIME} --conflict-exit-code 3 ${LOCKFD[${LOCK}]}
    ret=$?
    set -e
    case ${ret} in
        0)
            return
            ;;
        3)
263
            log_error "Could not get lock ${LOCKFILE}, timeout"
Joerg Jaspert's avatar
Joerg Jaspert committed
264
            laststeps 2
265 266
            ;;
        *)
267
            log_error "Could not get lock ${LOCKFILE}"
Joerg Jaspert's avatar
Joerg Jaspert committed
268
            laststeps 2
269
    esac
270 271 272 273 274 275 276 277 278 279 280 281 282
}

function unlock() {
    local LOCK=${1:-}
    if [[ -z ${LOCK} ]]; then
        # Warn, but continue, unlock will happen at script end time
        log "No lockfile name given"
    fi

    local randomstring=${LOCKFD[${LOCK}]}
    exec {randomstring}>&-
}

Joerg Jaspert's avatar
Joerg Jaspert committed
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
function laststeps() {
    local successval=${1:-0}

    # Redirect output to another file, as we want to compress our logfile
    # and ensure its no longer used
    exec > "$logdir/after${PROGRAM}.log" 2>&1

    # Now, at the very (successful) end of this run, make sure we remove
    # our stage files, so the next script run will do it all again.
    if [[ ${successval} -eq 0 ]]; then
        rm -f ${stagedir}/*
    fi
    bzip2 -9 ${LOGFILE}

    # Logfile should be gone, remove the symlink
    [[ -L ${logdir}/${PROGRAM} ]] && [[ ! -f ${logdir}/${PROGRAM} ]] && rm -f ${logdir}/${PROGRAM} || log "Logfile still exists or symlink gone already? Something fishy going on"

    # FIXME: Mail the log when its non-empty
    [[ -s "${logdir}/after${PROGRAM}.log" ]] || rm "${logdir}/after${PROGRAM}.log"
}

304
(
305
    # Where we store lockfile filehandles
306
    declare -A LOCKFD
307

308 309
    # common functions are "outsourced"
    . "${configdir}/common"
310

311 312
    # Timestamp when we started
    NOW=$(date "+%Y.%m.%d-%H:%M:%S")
313

314 315
    # A logfile for every cron script
    LOGFILE="${logdir}/${PROGRAM}_${NOW}.log"
316

317 318
    # Each "cronscript" may have a variables and a functions file
    # that we source
319
    _preparetasks ${PROGRAM}
320

321 322
    # Get rid of tempfiles at the end
    trap cleanup EXIT TERM HUP INT QUIT
323

324 325 326 327 328 329 330 331 332 333 334 335
    # An easy access by name for the current log
    ln -sf ${LOGFILE} ${logdir}/${PROGRAM}

    # And from here, all output to the log please
    exec >> "$LOGFILE" 2>&1

    # The stage function uses this directory
    # This amends the stagedir variable from "vars"
    stagedir="${stagedir}/${PROGRAM}"
    # Ensure the dir exists
    mkdir -p ${stagedir}

336 337
    # Run all tasks
    _runtasks ${PROGRAM}
338 339 340 341 342 343 344

    # we need to wait for the background processes before the end of the cron script
    wait

    # Common to all cron scripts
    log "Cron script successful, all done"

Joerg Jaspert's avatar
Joerg Jaspert committed
345
    laststeps 0
346
)