Commits (1)
  • Antoine Beaupré's avatar
    new check-advisories command · aecd1daa
    Antoine Beaupré authored
    This command makes sure that all advisories in the security tracker
    are regularly published on the website.
    It can be called in DSA or DLA mode and will (respectively) check for
    official security advisories or LTS advsiories on a local copy of the
    It does not check the integrity of that copy, it only checks to see if
    it's there.
    This is useful to make sure we don't forget to publish some advisories.
import argparse
import logging
import os.path
import re
import requests
# regex used in data/DLA/list and data/DSA/list files to identify
# advisories
RE_ADV = r'\[\d+\s+\w+\s+(?P<year>\d+)\]\s+D[SL]A-(?P<number>\d+)(?:-(?P<errata>\d+))?' # noqa: E501
RE_SKIP = r'^\s+'
# the URL for the lists if not provided locally, joker replaced with
# DSA or DLA depending on --mode
BASE_URL = 'https://salsa.debian.org/security-tracker-team/security-tracker/raw/master/data/%s/list' # noqa: E501
class LoggingAction(argparse.Action):
"""change log level on the fly
The logging system should be initialized befure this, using
def __init__(self, *args, **kwargs):
"""setup the action parameters
This enforces a selection of logging levels. It also checks if
const is provided, in which case we assume it's an argument
like `--verbose` or `--debug` without an argument.
kwargs['choices'] = logging._nameToLevel.keys()
if 'const' in kwargs:
kwargs['nargs'] = 0
super().__init__(*args, **kwargs)
def __call__(self, parser, ns, values, option):
"""if const was specified it means argument-less parameters"""
if self.const:
def main():
logging.basicConfig(format='%(levelname)s: %(message)s', level='WARNING')
parser = argparse.ArgumentParser(description='Find missing advisories')
parser.add_argument('--list', help='advisory list (default: fetch)')
help='website path (default: english/security or english/lts/security, depending on mode)') # noqa: E501
parser.add_argument('--verbose', action=LoggingAction, const='INFO',
help='show more progress information')
parser.add_argument('--debug', action=LoggingAction, const='DEBUG',
help='show debug information')
parser.add_argument('--mode', default='DSA', choices=('DSA', 'DLA'),
help='which sort of advisory to check (default: %(default)s)') # noqa: E501
args = parser.parse_args()
if not args.directory:
if args.mode == 'DSA':
args.directory = 'english/security'
elif args.mode == 'DLA':
args.directory = 'english/lts/security'
if not args.list:
url = BASE_URL % args.mode
logging.info('fetching URL %s', url)
response = requests.get(url)
for adv in parse_advisories(response.iter_lines(decode_unicode=True)):
check_advisory(args.mode, args.directory, **adv)
with open(args.list) as text:
for adv in parse_advisories(text):
check_advisory(args.mode, args.directory, **adv)
def parse_advisories(stream):
for line in stream:
m = re.match(RE_ADV, line)
if m:
yield m.groupdict()
elif re.match(RE_SKIP, line):
logging.debug('skipping line: "%s"', line)
logging.warning('malformed line: "%s"', line)
def check_advisory(mode, directory, year, number, errata):
if errata is None:
errata = '1'
logging.info('checking %s-%s-%s (%s)', mode, number, errata, year)
path = "%s/%s/%s-%s-%s" % (directory, year, mode.lower(), number, errata)
found = False
if os.path.exists(path + '.data') and os.path.exists(path + '.wml'):
logging.debug('both data and wml files found')
found = True
elif errata == '1':
path = "%s/%s/%s-%s" % (directory, year, mode.lower(), number)
if os.path.exists(path + '.data') and os.path.exists(path + '.wml'):
logging.debug('both data and wml files found, without -1')
found = True
if not found:
logging.error('.data or .wml file missing for %s %s-%s',
mode, number, errata)
if __name__ == '__main__':