diff --git a/bin/check-syntax b/bin/check-syntax index ee23752068cd6cde7e3abbd22f80ed50ed9bcdaf..475cf87d38559e7579b8a84e738c974ba275478a 100755 --- a/bin/check-syntax +++ b/bin/check-syntax @@ -65,13 +65,10 @@ def parse_DSA(name): def parse_DTSA(name): do_parse(construct(bugs.DTSAFile, name)) -def parse_DLA(name): - do_parse(construct(bugs.DLAFile, name)) - file_types = {'CVE' : parse_CVE, 'DSA' : parse_DSA, 'DTSA' : parse_DTSA, - 'DLA' : parse_DLA} + 'DLA' : parse_DSA} if len(sys.argv) <> 3 or not file_types.has_key(sys.argv[1]): l = file_types.keys() diff --git a/bin/gen-DSA b/bin/gen-DSA index 0eb389d51ccd8989ac170f88f10f6fa4224f6221..f2ba43e82bc58c6224a5ac68c761469efa54cf03 100755 --- a/bin/gen-DSA +++ b/bin/gen-DSA @@ -22,7 +22,7 @@ set -e IDMODE=DSA case "$(basename "$0")" in - *gen-D[LS]A) + *gen-*) IDMODE=${0#*gen-} ;; esac @@ -333,11 +333,7 @@ setvar DEBFULLNAME setvar SPACEDDEBFULLNAME setvar PACKAGE setvar CVE "$CVE_LIST" -if [ "$IDMODE" = DSA ]; then - setvar DSAID "$DAID" -else - setvar DLAID "$DAID" -fi +setvar ${IDMODE}ID "$DAID" setvar BUGNUM setvar OLDOLDSTABLE setvar OLDSTABLE diff --git a/data/config.json b/data/config.json index ee89ac614a7238ca2f16b7855b2cb7eef6352564..f59ee1b8d45ef51d43e7283fbcacdb484cc49658 100644 --- a/data/config.json +++ b/data/config.json @@ -30,13 +30,13 @@ "members" : { "supported" : ["lenny", "lenny-security"], "optional" : ["lenny-proposed-updates"] - }, + } }, "squeeze" : { "members" : { "supported" : ["squeeze", "squeeze-security"], "optional" : ["squeeze-proposed-updates"] - }, + } }, "wheezy" : { "members" : { @@ -65,5 +65,12 @@ }, "release" : "unstable" } + }, + + "sources" : { + "/CVE/list" : "CVEFile", + "/DSA/list" : "DSAFile", + "/DTSA/list" : "DTSAFile", + "/DLA/list" : "DSAFile" } } diff --git a/doc/security-team.d.o/security_tracker b/doc/security-team.d.o/security_tracker index eeea313ca8ad6a33e396826473dfa28c3e2abab8..cb91082bf7b5949bbe24f159d19b1a21aff4f1f1 100644 --- a/doc/security-team.d.o/security_tracker +++ b/doc/security-team.d.o/security_tracker @@ -612,3 +612,23 @@ The following commands build the databases for stable and run a python local ser make serve The website is now available as `http://127.0.0.1:10605/tracker/`. + +Setting up an extended instance +------------------------------- + +The security tracker supports extra sources of data, which can be used +to override or extend the information in CVE/list, and to support your +own announce lists. To do that, add a CVEExtendFile source to +`data/config.json`. Entries in that file can add information to an +existing CVE, e.g. to mark it as fixed or ignored, or to mark it as +affecting additional source packages. For example: + +CVE-2018-11646 + - webkitgtk <unfixed> +CVE-2016-1000340 + [wheezy] - bouncycastle <not-affected> (Vulnerable code introduced later) + +You can also add an announce list of type DSAFile to `data/config.json`, +and then symlink `bin/gen-DSA` to e.g. `bin/gen-MySA` and use that to +create new advisories under your namespace. For that you will need to +add a `data/mysa-needed.txt` file and `doc/MYSA.template`. diff --git a/lib/python/bugs.py b/lib/python/bugs.py index b876647da9c247f8be2cf7eb91848fe25eec7301..bcfa9b1ce575a2aa938d837f8a9f65822d453dfb 100644 --- a/lib/python/bugs.py +++ b/lib/python/bugs.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import debian_support +import os import re import types import hashlib @@ -200,6 +201,7 @@ class BugBase: self.notes = [] self.xref = [] self.not_for_us = False + self.is_extend = False def isFromCVE(self): """Returns True if the name has been officially assigned. @@ -225,16 +227,18 @@ class BugBase: not_for_us = 0 import apsw - try: - cursor.execute("""INSERT INTO bugs - (name, cve_status, not_for_us, description, release_date, - source_file, source_line) - VALUES (?, ?, ?, ?, ?, ?, ?)""", - (self.name, self.cveStatus(), not_for_us, - self.description, self.date or '', - self.source_file, self.source_line)) - except apsw.ConstraintError: - raise ValueError, "bug name %s is not unique" % self.name + + if not self.is_extend: + try: + cursor.execute("""INSERT INTO bugs + (name, cve_status, not_for_us, description, release_date, + source_file, source_line) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (self.name, self.cveStatus(), not_for_us, + self.description, self.date or '', + self.source_file, self.source_line)) + except apsw.ConstraintError: + raise ValueError, "bug name %s is not unique" % self.name for (typ, c) in self.comments: cursor.execute("""INSERT INTO bugs_notes @@ -257,7 +261,7 @@ class Bug(BugBase): """Class for bugs for which we have some data.""" def __init__(self, fname, lineno, date, name, description, comments, notes, - xref, not_for_us=False): + xref, not_for_us=False, is_extend=False): for n in notes: assert isinstance(n, PackageNote) \ or isinstance(n, PackageNoteNoDSA) @@ -268,6 +272,7 @@ class Bug(BugBase): self.notes = notes self.xref = xref self.not_for_us = not_for_us + self.is_extend = is_extend def mergeNotes(self): """Merge notes so that there is only one note for each @@ -435,6 +440,7 @@ class FileBase(debian_support.PackageFile): re_rejected = re.compile(r'^(?:NOTE:\s+rejected|REJECTED)\s*$') re_note = re.compile(r'^NOTE:\s+(.*)$') re_todo = re.compile(r'^TODO:\s+(.*)$') + is_extend = False def __init__(self, name, fileObj=None): debian_support.PackageFile.__init__(self, name, fileObj) @@ -730,7 +736,8 @@ class FileBase(debian_support.PackageFile): record_name = temp_bug_name(first_bug, description) yield self.finishBug(Bug(self.file.name, first_lineno, date, record_name, description, - comments, notes=pkg_notes, xref=xref)) + comments, notes=pkg_notes, xref=xref, + is_extend=self.is_extend)) def finishBug(self, bug): """Applies a transformation to the bug after it has been @@ -773,6 +780,15 @@ class CVEFile(FileBase): bug.mergeNotes() return bug +class CVEExtendFile(CVEFile): + # This is an extend file. The main CVEFile can have a 'CVE-2018-XXXX' (sic) + # identifier, which will get converted to TEMP-* automatically. However to + # refer to that one from here, we need to use the TEMP-* identifier, so we + # allow those in the regex + re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$') + + is_extend = True + class DSAFile(FileBase): """A DSA file. @@ -780,9 +796,13 @@ class DSAFile(FileBase): reference point, and release dates. """ - re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' - + r'(DSA-\d+(?:-\d+)?)\s+' - + r'(.*?)\s*$') + def __init__(self, name, fileObj=None): + FileBase.__init__(self, name, fileObj) + + self.base = os.path.basename(os.path.dirname(self.name)) + self.re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' + + r'(' + self.base + '-\d+(?:-\d+)?)\s+' + + r'(.*?)\s*$') month_names = {'Jan': 1, 'Feb': 2, @@ -800,7 +820,7 @@ class DSAFile(FileBase): def matchHeader(self, line): match = self.re_dsa.match(line) if not match: - self.raiseSyntaxError("expected DSA record, got: %s" % `line`) + self.raiseSyntaxError("expected %s record, got: %s" % (self.base, `line`)) (record_name, description) = match.groups() (day, month, year, name, desc) = match.groups() try: @@ -814,47 +834,7 @@ class DSAFile(FileBase): bug.mergeNotes() return bug -class DLAFile(FileBase): - """A DLA file. - Similar to a CVE file, only that it contains DLAs as its main - reference point, and release dates. - """ - - re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' - + r'(DLA-\d+(?:-\d+)?)\s+' - + r'(.*?)\s*$') - - month_names = {'Jan': 1, - 'Feb': 2, - 'Mar': 3, - 'Apr': 4, - 'May': 5, - 'Jun': 6, - 'Jul': 7, - 'Aug': 8, - 'Sep': 9, - 'Oct': 10, - 'Nov': 11, - 'Dec': 12} - - def matchHeader(self, line): - match = self.re_dsa.match(line) - if not match: - self.raiseSyntaxError("expected DLA record, got: %s" % `line`) - (record_name, description) = match.groups() - (day, month, year, name, desc) = match.groups() - try: - month = self.month_names[month] - except KeyError: - self.raiseSyntaxError("invalid month name %s" % `month`) - return ("%s-%02d-%s" % (year, month, day), name, desc) - - def finishBug(self, bug): - # Merge identical package notes, for historical reasons. - bug.mergeNotes() - return bug - class DTSAFile(FileBase): """A DTSA file. diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 9208532fbaff8181c3d3ae6ec4c2857a14304c5d..9bcbb3eaecb078badb629efdd427a803f99b275a 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -856,6 +856,28 @@ class DB: VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", gen()) + def getSources(self): + config = debian_support.getconfig() + sources = config["sources"] + + return sources + + def genDBAdvisoryString(self, field, dtsa=False): + sources = self.getSources() + advs = [] + + for path, cls in sources.iteritems(): + name = path.split('/')[1] + + if cls == 'DSAFile': + advs.append(name) + + if cls == 'DTSAFile' and dtsa: + advs.append(name) + + advs = ["{} LIKE '{}-%'".format(field, adv) for adv in advs] + return " OR ".join(advs) + def readBugs(self, cursor, path): if self.verbose: print "readBugs:" @@ -913,15 +935,11 @@ class DB: return True source_removed_packages = '/packages/removed-packages' - sources = ((bugs.CVEFile, '/CVE/list'), - (bugs.DSAFile, '/DSA/list'), - (bugs.DTSAFile, '/DTSA/list'), - (bugs.DLAFile, '/DLA/list'), - (None, source_removed_packages)) + sources = self.getSources() unchanged = True - for (_, name) in sources: - if has_changed(path + name): + for filename in sources.keys() + [source_removed_packages]: + if has_changed(path + filename): unchanged = False break if unchanged: @@ -940,9 +958,8 @@ class DB: """INSERT OR REPLACE INTO inodeprints (inodeprint, file) VALUES (?, ?)""", (current_print, filename)) - for (cls, name) in sources: - if cls is None: - continue + for name, cls in sources.iteritems(): + cls = getattr(bugs, cls) read_one(cls(path + name)) if self.verbose: @@ -967,9 +984,10 @@ class DB: # Copy notes from DSA/DTSA/DLA to CVE. old_source = '' + source_like = self.genDBAdvisoryString("source", dtsa=True) for source, target in list(cursor.execute( """SELECT source, target FROM bugs_xref - WHERE (source LIKE 'DTSA-%' OR source LIKE 'DSA-%' OR source LIKE 'DLA-%') + WHERE (""" + source_like + """) AND target LIKE 'CVE-%'""")): if source <> old_source: source_bug = bugs.BugFromDB(cursor, source) @@ -1848,11 +1866,12 @@ class DB: return flag def getDSAsForSourcePackage(self, cursor, package): + bugs_like = self.genDBAdvisoryString("bugs.name", dtsa=False) for row in cursor.execute( """SELECT bugs.name, bugs.description FROM bugs, package_notes as p WHERE p.bug_name = bugs.name - AND ( bugs.name LIKE 'DSA-%' OR bugs.name LIKE 'DLA-%') + AND ( """ + bugs_like + """ ) AND p.package = ? ORDER BY bugs.release_date DESC""", (package,)): yield DSAsForSourcePackage(*row)