From 0b664197d33a0e35348969561dafaad09b6c44a0 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 11 May 2018 15:35:39 +0200 Subject: [PATCH 1/9] Simplify DLAFile Subclass DSAFile rather than copying it. --- lib/python/bugs.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/lib/python/bugs.py b/lib/python/bugs.py index b876647da9c..6a042033b54 100644 --- a/lib/python/bugs.py +++ b/lib/python/bugs.py @@ -814,47 +814,12 @@ class DSAFile(FileBase): bug.mergeNotes() return bug -class DLAFile(FileBase): - """A DLA file. - - Similar to a CVE file, only that it contains DLAs as its main - reference point, and release dates. - """ - +class DLAFile(DSAFile): re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' + r'(DLA-\d+(?:-\d+)?)\s+' + r'(.*?)\s*$') - month_names = {'Jan': 1, - 'Feb': 2, - 'Mar': 3, - 'Apr': 4, - 'May': 5, - 'Jun': 6, - 'Jul': 7, - 'Aug': 8, - 'Sep': 9, - 'Oct': 10, - 'Nov': 11, - 'Dec': 12} - def matchHeader(self, line): - match = self.re_dsa.match(line) - if not match: - self.raiseSyntaxError("expected DLA record, got: %s" % `line`) - (record_name, description) = match.groups() - (day, month, year, name, desc) = match.groups() - try: - month = self.month_names[month] - except KeyError: - self.raiseSyntaxError("invalid month name %s" % `month`) - return ("%s-%02d-%s" % (year, month, day), name, desc) - - def finishBug(self, bug): - # Merge identical package notes, for historical reasons. - bug.mergeNotes() - return bug - class DTSAFile(FileBase): """A DTSA file. -- GitLab From 375ba023951257874416fd6fc291945210354d66 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 1 Jun 2018 15:38:19 +0200 Subject: [PATCH 2/9] Merge DLAFile into DSAFile The only difference is that the regular expressions look for DSA or DLA, but we can just guess that based on the path. --- bin/check-syntax | 5 +---- lib/python/bugs.py | 18 +++++++++--------- lib/python/security_db.py | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bin/check-syntax b/bin/check-syntax index ee23752068c..475cf87d385 100755 --- a/bin/check-syntax +++ b/bin/check-syntax @@ -65,13 +65,10 @@ def parse_DSA(name): def parse_DTSA(name): do_parse(construct(bugs.DTSAFile, name)) -def parse_DLA(name): - do_parse(construct(bugs.DLAFile, name)) - file_types = {'CVE' : parse_CVE, 'DSA' : parse_DSA, 'DTSA' : parse_DTSA, - 'DLA' : parse_DLA} + 'DLA' : parse_DSA} if len(sys.argv) <> 3 or not file_types.has_key(sys.argv[1]): l = file_types.keys() diff --git a/lib/python/bugs.py b/lib/python/bugs.py index 6a042033b54..278e2ab79e8 100644 --- a/lib/python/bugs.py +++ b/lib/python/bugs.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import debian_support +import os import re import types import hashlib @@ -780,9 +781,13 @@ class DSAFile(FileBase): reference point, and release dates. """ - re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' - + r'(DSA-\d+(?:-\d+)?)\s+' - + r'(.*?)\s*$') + def __init__(self, name, fileObj=None): + FileBase.__init__(self, name, fileObj) + + self.base = os.path.basename(os.path.dirname(self.name)) + self.re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' + + r'(' + self.base + '-\d+(?:-\d+)?)\s+' + + r'(.*?)\s*$') month_names = {'Jan': 1, 'Feb': 2, @@ -800,7 +805,7 @@ class DSAFile(FileBase): def matchHeader(self, line): match = self.re_dsa.match(line) if not match: - self.raiseSyntaxError("expected DSA record, got: %s" % `line`) + self.raiseSyntaxError("expected %s record, got: %s" % (self.base, `line`)) (record_name, description) = match.groups() (day, month, year, name, desc) = match.groups() try: @@ -814,11 +819,6 @@ class DSAFile(FileBase): bug.mergeNotes() return bug -class DLAFile(DSAFile): - re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] ' - + r'(DLA-\d+(?:-\d+)?)\s+' - + r'(.*?)\s*$') - class DTSAFile(FileBase): """A DTSA file. diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 9208532fbaf..dbb7cd8048c 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -916,7 +916,7 @@ class DB: sources = ((bugs.CVEFile, '/CVE/list'), (bugs.DSAFile, '/DSA/list'), (bugs.DTSAFile, '/DTSA/list'), - (bugs.DLAFile, '/DLA/list'), + (bugs.DSAFile, '/DLA/list'), (None, source_removed_packages)) unchanged = True -- GitLab From 0cb94dee777219c6edf3b3c2579c17fe46afc13f Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 1 Jun 2018 15:24:58 +0200 Subject: [PATCH 3/9] Move source list to a config file --- data/config.json | 11 +++++++++-- lib/python/security_db.py | 21 +++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/data/config.json b/data/config.json index ee89ac614a7..f59ee1b8d45 100644 --- a/data/config.json +++ b/data/config.json @@ -30,13 +30,13 @@ "members" : { "supported" : ["lenny", "lenny-security"], "optional" : ["lenny-proposed-updates"] - }, + } }, "squeeze" : { "members" : { "supported" : ["squeeze", "squeeze-security"], "optional" : ["squeeze-proposed-updates"] - }, + } }, "wheezy" : { "members" : { @@ -65,5 +65,12 @@ }, "release" : "unstable" } + }, + + "sources" : { + "/CVE/list" : "CVEFile", + "/DSA/list" : "DSAFile", + "/DTSA/list" : "DTSAFile", + "/DLA/list" : "DSAFile" } } diff --git a/lib/python/security_db.py b/lib/python/security_db.py index dbb7cd8048c..9c7d4afd8b1 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -856,6 +856,12 @@ class DB: VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", gen()) + def getSources(self): + config = debian_support.getconfig() + sources = config["sources"] + + return sources + def readBugs(self, cursor, path): if self.verbose: print "readBugs:" @@ -913,15 +919,11 @@ class DB: return True source_removed_packages = '/packages/removed-packages' - sources = ((bugs.CVEFile, '/CVE/list'), - (bugs.DSAFile, '/DSA/list'), - (bugs.DTSAFile, '/DTSA/list'), - (bugs.DSAFile, '/DLA/list'), - (None, source_removed_packages)) + sources = self.getSources() unchanged = True - for (_, name) in sources: - if has_changed(path + name): + for filename in sources.keys() + [source_removed_packages]: + if has_changed(path + filename): unchanged = False break if unchanged: @@ -940,9 +942,8 @@ class DB: """INSERT OR REPLACE INTO inodeprints (inodeprint, file) VALUES (?, ?)""", (current_print, filename)) - for (cls, name) in sources: - if cls is None: - continue + for name, cls in sources.iteritems(): + cls = getattr(bugs, cls) read_one(cls(path + name)) if self.verbose: -- GitLab From 77190d32d7f00156830d7fbefe2df1e483b194e9 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 1 Jun 2018 13:52:53 +0200 Subject: [PATCH 4/9] Dynamically create announce queries Based on the DSA-like files present in the config file. --- lib/python/security_db.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/python/security_db.py b/lib/python/security_db.py index 9c7d4afd8b1..9bcbb3eaecb 100644 --- a/lib/python/security_db.py +++ b/lib/python/security_db.py @@ -862,6 +862,22 @@ class DB: return sources + def genDBAdvisoryString(self, field, dtsa=False): + sources = self.getSources() + advs = [] + + for path, cls in sources.iteritems(): + name = path.split('/')[1] + + if cls == 'DSAFile': + advs.append(name) + + if cls == 'DTSAFile' and dtsa: + advs.append(name) + + advs = ["{} LIKE '{}-%'".format(field, adv) for adv in advs] + return " OR ".join(advs) + def readBugs(self, cursor, path): if self.verbose: print "readBugs:" @@ -968,9 +984,10 @@ class DB: # Copy notes from DSA/DTSA/DLA to CVE. old_source = '' + source_like = self.genDBAdvisoryString("source", dtsa=True) for source, target in list(cursor.execute( """SELECT source, target FROM bugs_xref - WHERE (source LIKE 'DTSA-%' OR source LIKE 'DSA-%' OR source LIKE 'DLA-%') + WHERE (""" + source_like + """) AND target LIKE 'CVE-%'""")): if source <> old_source: source_bug = bugs.BugFromDB(cursor, source) @@ -1849,11 +1866,12 @@ class DB: return flag def getDSAsForSourcePackage(self, cursor, package): + bugs_like = self.genDBAdvisoryString("bugs.name", dtsa=False) for row in cursor.execute( """SELECT bugs.name, bugs.description FROM bugs, package_notes as p WHERE p.bug_name = bugs.name - AND ( bugs.name LIKE 'DSA-%' OR bugs.name LIKE 'DLA-%') + AND ( """ + bugs_like + """ ) AND p.package = ? ORDER BY bugs.release_date DESC""", (package,)): yield DSAsForSourcePackage(*row) -- GitLab From ecbbab3e6af9d16c94d559db65ad326568b3ad12 Mon Sep 17 00:00:00 2001 From: Bastian Blank <bastian.blank@credativ.de> Date: Wed, 23 Mar 2016 10:51:02 +0100 Subject: [PATCH 5/9] Add support for CUSTOMER bugs and CVE extends --- lib/python/bugs.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/python/bugs.py b/lib/python/bugs.py index 278e2ab79e8..5ec843b6218 100644 --- a/lib/python/bugs.py +++ b/lib/python/bugs.py @@ -300,6 +300,28 @@ class Bug(BugBase): nts.append(notes[key]) self.notes = nts +class BugExtend(Bug): + def writeDB(self, cursor): + """Writes the record to an SQLite3 database.""" + + for (typ, c) in self.comments: + cursor.execute("""INSERT INTO bugs_notes + (bug_name, typ, comment) VALUES (?, ?, ?)""", + (self.name, typ, c)) + + for n in self.notes: + n.writeDB(cursor, self.name) + + import apsw + for x in self.xref: + try: + cursor.execute("""INSERT INTO bugs_xref + (source, target) VALUES (?, ?)""", + (self.name, x)) + except apsw.ConstraintError: + raise ValueError, \ + "cross reference to %s appears multiple times" % x + class BugFromDB(Bug): def __init__(self, cursor, name): assert type(name) in types.StringTypes @@ -441,6 +463,9 @@ class FileBase(debian_support.PackageFile): debian_support.PackageFile.__init__(self, name, fileObj) self.removed_packages = {} + def isExtend(self, name): + return False + def isUniqueName(self, name): """Returns True if the name is a real, unique name.""" return True @@ -729,7 +754,11 @@ class FileBase(debian_support.PackageFile): if first_bug: break record_name = temp_bug_name(first_bug, description) - yield self.finishBug(Bug(self.file.name, first_lineno, date, + if self.isExtend(record_name): + cls = BugExtend + else: + cls = Bug + yield self.finishBug(cls(self.file.name, first_lineno, date, record_name, description, comments, notes=pkg_notes, xref=xref)) @@ -774,6 +803,12 @@ class CVEFile(FileBase): bug.mergeNotes() return bug +class CVECUSTOMERFile(CVEFile): + re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$') + + def isExtend(self, name): + return True + class DSAFile(FileBase): """A DSA file. -- GitLab From fafe483991149607160325eb6ef572faa141acf2 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 11 May 2018 15:22:31 +0200 Subject: [PATCH 6/9] Simplify Extends support BugExtend.writeDB() is pretty similar to BugBase's, so update the latter to take extends into account when necessary to avoid unneeded duplicated code. --- lib/python/bugs.py | 64 +++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 44 deletions(-) diff --git a/lib/python/bugs.py b/lib/python/bugs.py index 5ec843b6218..676f5f540f3 100644 --- a/lib/python/bugs.py +++ b/lib/python/bugs.py @@ -201,6 +201,7 @@ class BugBase: self.notes = [] self.xref = [] self.not_for_us = False + self.is_extend = False def isFromCVE(self): """Returns True if the name has been officially assigned. @@ -226,16 +227,18 @@ class BugBase: not_for_us = 0 import apsw - try: - cursor.execute("""INSERT INTO bugs - (name, cve_status, not_for_us, description, release_date, - source_file, source_line) - VALUES (?, ?, ?, ?, ?, ?, ?)""", - (self.name, self.cveStatus(), not_for_us, - self.description, self.date or '', - self.source_file, self.source_line)) - except apsw.ConstraintError: - raise ValueError, "bug name %s is not unique" % self.name + + if not self.is_extend: + try: + cursor.execute("""INSERT INTO bugs + (name, cve_status, not_for_us, description, release_date, + source_file, source_line) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (self.name, self.cveStatus(), not_for_us, + self.description, self.date or '', + self.source_file, self.source_line)) + except apsw.ConstraintError: + raise ValueError, "bug name %s is not unique" % self.name for (typ, c) in self.comments: cursor.execute("""INSERT INTO bugs_notes @@ -258,7 +261,7 @@ class Bug(BugBase): """Class for bugs for which we have some data.""" def __init__(self, fname, lineno, date, name, description, comments, notes, - xref, not_for_us=False): + xref, not_for_us=False, is_extend=False): for n in notes: assert isinstance(n, PackageNote) \ or isinstance(n, PackageNoteNoDSA) @@ -269,6 +272,7 @@ class Bug(BugBase): self.notes = notes self.xref = xref self.not_for_us = not_for_us + self.is_extend = is_extend def mergeNotes(self): """Merge notes so that there is only one note for each @@ -300,28 +304,6 @@ class Bug(BugBase): nts.append(notes[key]) self.notes = nts -class BugExtend(Bug): - def writeDB(self, cursor): - """Writes the record to an SQLite3 database.""" - - for (typ, c) in self.comments: - cursor.execute("""INSERT INTO bugs_notes - (bug_name, typ, comment) VALUES (?, ?, ?)""", - (self.name, typ, c)) - - for n in self.notes: - n.writeDB(cursor, self.name) - - import apsw - for x in self.xref: - try: - cursor.execute("""INSERT INTO bugs_xref - (source, target) VALUES (?, ?)""", - (self.name, x)) - except apsw.ConstraintError: - raise ValueError, \ - "cross reference to %s appears multiple times" % x - class BugFromDB(Bug): def __init__(self, cursor, name): assert type(name) in types.StringTypes @@ -458,14 +440,12 @@ class FileBase(debian_support.PackageFile): re_rejected = re.compile(r'^(?:NOTE:\s+rejected|REJECTED)\s*$') re_note = re.compile(r'^NOTE:\s+(.*)$') re_todo = re.compile(r'^TODO:\s+(.*)$') + is_extend = False def __init__(self, name, fileObj=None): debian_support.PackageFile.__init__(self, name, fileObj) self.removed_packages = {} - def isExtend(self, name): - return False - def isUniqueName(self, name): """Returns True if the name is a real, unique name.""" return True @@ -754,13 +734,10 @@ class FileBase(debian_support.PackageFile): if first_bug: break record_name = temp_bug_name(first_bug, description) - if self.isExtend(record_name): - cls = BugExtend - else: - cls = Bug - yield self.finishBug(cls(self.file.name, first_lineno, date, + yield self.finishBug(Bug(self.file.name, first_lineno, date, record_name, description, - comments, notes=pkg_notes, xref=xref)) + comments, notes=pkg_notes, xref=xref, + is_extend=self.is_extend)) def finishBug(self, bug): """Applies a transformation to the bug after it has been @@ -806,8 +783,7 @@ class CVEFile(FileBase): class CVECUSTOMERFile(CVEFile): re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$') - def isExtend(self, name): - return True + is_extend = True class DSAFile(FileBase): """A DSA file. -- GitLab From bf1f037a3270e19cda4fb350124fc21f3964f6c1 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 11 May 2018 16:18:06 +0200 Subject: [PATCH 7/9] gen-DSA: allow other gen-* links --- bin/gen-DSA | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/gen-DSA b/bin/gen-DSA index 0eb389d51cc..f2ba43e82bc 100755 --- a/bin/gen-DSA +++ b/bin/gen-DSA @@ -22,7 +22,7 @@ set -e IDMODE=DSA case "$(basename "$0")" in - *gen-D[LS]A) + *gen-*) IDMODE=${0#*gen-} ;; esac @@ -333,11 +333,7 @@ setvar DEBFULLNAME setvar SPACEDDEBFULLNAME setvar PACKAGE setvar CVE "$CVE_LIST" -if [ "$IDMODE" = DSA ]; then - setvar DSAID "$DAID" -else - setvar DLAID "$DAID" -fi +setvar ${IDMODE}ID "$DAID" setvar BUGNUM setvar OLDOLDSTABLE setvar OLDSTABLE -- GitLab From b59cbe46e12d468b4a41cf599ca12a2dcd024b92 Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 11 May 2018 17:51:01 +0200 Subject: [PATCH 8/9] Rename CVECUSTOMERFile to CVEExtendFile --- lib/python/bugs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/python/bugs.py b/lib/python/bugs.py index 676f5f540f3..bcfa9b1ce57 100644 --- a/lib/python/bugs.py +++ b/lib/python/bugs.py @@ -780,7 +780,11 @@ class CVEFile(FileBase): bug.mergeNotes() return bug -class CVECUSTOMERFile(CVEFile): +class CVEExtendFile(CVEFile): + # This is an extend file. The main CVEFile can have a 'CVE-2018-XXXX' (sic) + # identifier, which will get converted to TEMP-* automatically. However to + # refer to that one from here, we need to use the TEMP-* identifier, so we + # allow those in the regex re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$') is_extend = True -- GitLab From a0c205800723e54d383c1cb54969e4da3922edfb Mon Sep 17 00:00:00 2001 From: Emilio Pozuelo Monfort <pochu@debian.org> Date: Fri, 8 Jun 2018 09:46:29 +0200 Subject: [PATCH 9/9] Document CVE extends support --- doc/security-team.d.o/security_tracker | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/security-team.d.o/security_tracker b/doc/security-team.d.o/security_tracker index eeea313ca8a..cb91082bf7b 100644 --- a/doc/security-team.d.o/security_tracker +++ b/doc/security-team.d.o/security_tracker @@ -612,3 +612,23 @@ The following commands build the databases for stable and run a python local ser make serve The website is now available as `http://127.0.0.1:10605/tracker/`. + +Setting up an extended instance +------------------------------- + +The security tracker supports extra sources of data, which can be used +to override or extend the information in CVE/list, and to support your +own announce lists. To do that, add a CVEExtendFile source to +`data/config.json`. Entries in that file can add information to an +existing CVE, e.g. to mark it as fixed or ignored, or to mark it as +affecting additional source packages. For example: + +CVE-2018-11646 + - webkitgtk <unfixed> +CVE-2016-1000340 + [wheezy] - bouncycastle <not-affected> (Vulnerable code introduced later) + +You can also add an announce list of type DSAFile to `data/config.json`, +and then symlink `bin/gen-DSA` to e.g. `bin/gen-MySA` and use that to +create new advisories under your namespace. For that you will need to +add a `data/mysa-needed.txt` file and `doc/MYSA.template`. -- GitLab