From 0b664197d33a0e35348969561dafaad09b6c44a0 Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 11 May 2018 15:35:39 +0200
Subject: [PATCH 1/9] Simplify DLAFile

Subclass DSAFile rather than copying it.
---
 lib/python/bugs.py | 37 +------------------------------------
 1 file changed, 1 insertion(+), 36 deletions(-)

diff --git a/lib/python/bugs.py b/lib/python/bugs.py
index b876647da9c..6a042033b54 100644
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -814,47 +814,12 @@ class DSAFile(FileBase):
         bug.mergeNotes()
         return bug
 
-class DLAFile(FileBase):
-    """A DLA file.
-
-    Similar to a CVE file, only that it contains DLAs as its main
-    reference point, and release dates.
-    """
-
+class DLAFile(DSAFile):
     re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
                         + r'(DLA-\d+(?:-\d+)?)\s+'
                         + r'(.*?)\s*$')
 
-    month_names = {'Jan': 1,
-                   'Feb': 2,
-                   'Mar': 3,
-                   'Apr': 4,
-                   'May': 5,
-                   'Jun': 6,
-                   'Jul': 7,
-                   'Aug': 8,
-                   'Sep': 9,
-                   'Oct': 10,
-                   'Nov': 11,
-                   'Dec': 12}
 
-    def matchHeader(self, line):
-        match = self.re_dsa.match(line)
-        if not match:
-            self.raiseSyntaxError("expected DLA record, got: %s" % `line`)
-            (record_name, description) = match.groups()
-        (day, month, year, name, desc) = match.groups()
-        try:
-            month = self.month_names[month]
-        except KeyError:
-            self.raiseSyntaxError("invalid month name %s" % `month`)
-        return ("%s-%02d-%s" % (year, month, day), name, desc)
-
-    def finishBug(self, bug):
-        # Merge identical package notes, for historical reasons.
-        bug.mergeNotes()
-        return bug
-         
 class DTSAFile(FileBase):
     """A DTSA file.
 
-- 
GitLab


From 375ba023951257874416fd6fc291945210354d66 Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 1 Jun 2018 15:38:19 +0200
Subject: [PATCH 2/9] Merge DLAFile into DSAFile

The only difference is that the regular expressions look for DSA
or DLA, but we can just guess that based on the path.
---
 bin/check-syntax          |  5 +----
 lib/python/bugs.py        | 18 +++++++++---------
 lib/python/security_db.py |  2 +-
 3 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/bin/check-syntax b/bin/check-syntax
index ee23752068c..475cf87d385 100755
--- a/bin/check-syntax
+++ b/bin/check-syntax
@@ -65,13 +65,10 @@ def parse_DSA(name):
 def parse_DTSA(name):
     do_parse(construct(bugs.DTSAFile, name))
 
-def parse_DLA(name):
-    do_parse(construct(bugs.DLAFile, name))
-
 file_types = {'CVE' : parse_CVE,
               'DSA' : parse_DSA,
               'DTSA' : parse_DTSA,
-              'DLA' : parse_DLA}
+              'DLA' : parse_DSA}
 
 if len(sys.argv) <> 3 or not file_types.has_key(sys.argv[1]):
     l = file_types.keys()
diff --git a/lib/python/bugs.py b/lib/python/bugs.py
index 6a042033b54..278e2ab79e8 100644
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -16,6 +16,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
 import debian_support
+import os
 import re
 import types
 import hashlib
@@ -780,9 +781,13 @@ class DSAFile(FileBase):
     reference point, and release dates.
     """
 
-    re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
-                        + r'(DSA-\d+(?:-\d+)?)\s+'
-                        + r'(.*?)\s*$')
+    def __init__(self, name, fileObj=None):
+        FileBase.__init__(self, name, fileObj)
+
+        self.base = os.path.basename(os.path.dirname(self.name))
+        self.re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
+                                + r'(' + self.base + '-\d+(?:-\d+)?)\s+'
+                                + r'(.*?)\s*$')
 
     month_names = {'Jan': 1,
                    'Feb': 2,
@@ -800,7 +805,7 @@ class DSAFile(FileBase):
     def matchHeader(self, line):
         match = self.re_dsa.match(line)
         if not match:
-            self.raiseSyntaxError("expected DSA record, got: %s" % `line`)
+            self.raiseSyntaxError("expected %s record, got: %s" % (self.base, `line`))
             (record_name, description) = match.groups()
         (day, month, year, name, desc) = match.groups()
         try:
@@ -814,11 +819,6 @@ class DSAFile(FileBase):
         bug.mergeNotes()
         return bug
 
-class DLAFile(DSAFile):
-    re_dsa = re.compile(r'^\[(\d\d) ([A-Z][a-z][a-z]) (\d{4})\] '
-                        + r'(DLA-\d+(?:-\d+)?)\s+'
-                        + r'(.*?)\s*$')
-
 
 class DTSAFile(FileBase):
     """A DTSA file.
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 9208532fbaf..dbb7cd8048c 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -916,7 +916,7 @@ class DB:
         sources = ((bugs.CVEFile, '/CVE/list'),
                    (bugs.DSAFile, '/DSA/list'),
                    (bugs.DTSAFile, '/DTSA/list'),
-                   (bugs.DLAFile, '/DLA/list'),
+                   (bugs.DSAFile, '/DLA/list'),
                    (None, source_removed_packages))
 
         unchanged = True
-- 
GitLab


From 0cb94dee777219c6edf3b3c2579c17fe46afc13f Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 1 Jun 2018 15:24:58 +0200
Subject: [PATCH 3/9] Move source list to a config file

---
 data/config.json          | 11 +++++++++--
 lib/python/security_db.py | 21 +++++++++++----------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/data/config.json b/data/config.json
index ee89ac614a7..f59ee1b8d45 100644
--- a/data/config.json
+++ b/data/config.json
@@ -30,13 +30,13 @@
       "members" : {
 	"supported" : ["lenny", "lenny-security"],
 	"optional" : ["lenny-proposed-updates"]
-      },
+      }
     },
     "squeeze" : {
       "members" : {
 	"supported" : ["squeeze", "squeeze-security"],
 	"optional" : ["squeeze-proposed-updates"]
-      },
+      }
     },
     "wheezy" : {
       "members" : {
@@ -65,5 +65,12 @@
       },
       "release" : "unstable"
     }
+  },
+
+  "sources" : {
+    "/CVE/list" : "CVEFile",
+    "/DSA/list" : "DSAFile",
+    "/DTSA/list" : "DTSAFile",
+    "/DLA/list" : "DSAFile"
   }
 }
diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index dbb7cd8048c..9c7d4afd8b1 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -856,6 +856,12 @@ class DB:
             VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
             gen())
 
+    def getSources(self):
+        config = debian_support.getconfig()
+        sources = config["sources"]
+
+        return sources
+
     def readBugs(self, cursor, path):
         if self.verbose:
             print "readBugs:"
@@ -913,15 +919,11 @@ class DB:
             return True
 
         source_removed_packages = '/packages/removed-packages'
-        sources = ((bugs.CVEFile, '/CVE/list'),
-                   (bugs.DSAFile, '/DSA/list'),
-                   (bugs.DTSAFile, '/DTSA/list'),
-                   (bugs.DSAFile, '/DLA/list'),
-                   (None, source_removed_packages))
+        sources = self.getSources()
 
         unchanged = True
-        for (_, name) in sources:
-            if has_changed(path + name):
+        for filename in sources.keys() + [source_removed_packages]:
+            if has_changed(path + filename):
                 unchanged = False
                 break
         if unchanged:
@@ -940,9 +942,8 @@ class DB:
                 """INSERT OR REPLACE INTO inodeprints (inodeprint, file)
                 VALUES (?, ?)""", (current_print, filename))
 
-        for (cls, name) in sources:
-            if cls is None:
-                continue
+        for name, cls in sources.iteritems():
+            cls = getattr(bugs, cls)
             read_one(cls(path + name))
 
         if self.verbose:
-- 
GitLab


From 77190d32d7f00156830d7fbefe2df1e483b194e9 Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 1 Jun 2018 13:52:53 +0200
Subject: [PATCH 4/9] Dynamically create announce queries

Based on the DSA-like files present in the config file.
---
 lib/python/security_db.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/lib/python/security_db.py b/lib/python/security_db.py
index 9c7d4afd8b1..9bcbb3eaecb 100644
--- a/lib/python/security_db.py
+++ b/lib/python/security_db.py
@@ -862,6 +862,22 @@ class DB:
 
         return sources
 
+    def genDBAdvisoryString(self, field, dtsa=False):
+        sources = self.getSources()
+        advs = []
+
+        for path, cls in sources.iteritems():
+            name = path.split('/')[1]
+
+            if cls == 'DSAFile':
+                advs.append(name)
+
+            if cls == 'DTSAFile' and dtsa:
+                advs.append(name)
+
+        advs = ["{} LIKE '{}-%'".format(field, adv) for adv in advs]
+        return " OR ".join(advs)
+
     def readBugs(self, cursor, path):
         if self.verbose:
             print "readBugs:"
@@ -968,9 +984,10 @@ class DB:
         # Copy notes from DSA/DTSA/DLA to CVE.
 
         old_source = ''
+        source_like = self.genDBAdvisoryString("source", dtsa=True)
         for source, target in list(cursor.execute(
             """SELECT source, target FROM bugs_xref
-            WHERE (source LIKE 'DTSA-%' OR source LIKE 'DSA-%' OR source LIKE 'DLA-%')
+            WHERE (""" + source_like +  """)
             AND target LIKE 'CVE-%'""")):
             if source <> old_source:
                 source_bug = bugs.BugFromDB(cursor, source)
@@ -1849,11 +1866,12 @@ class DB:
         return flag
 
     def getDSAsForSourcePackage(self, cursor, package):
+        bugs_like = self.genDBAdvisoryString("bugs.name", dtsa=False)
         for row in cursor.execute(
             """SELECT bugs.name, bugs.description
             FROM bugs, package_notes as p
             WHERE p.bug_name = bugs.name
-            AND ( bugs.name LIKE 'DSA-%' OR bugs.name LIKE 'DLA-%')
+            AND ( """ + bugs_like + """ )
             AND p.package = ?
             ORDER BY bugs.release_date DESC""", (package,)):
             yield DSAsForSourcePackage(*row)
-- 
GitLab


From ecbbab3e6af9d16c94d559db65ad326568b3ad12 Mon Sep 17 00:00:00 2001
From: Bastian Blank <bastian.blank@credativ.de>
Date: Wed, 23 Mar 2016 10:51:02 +0100
Subject: [PATCH 5/9] Add support for CUSTOMER bugs and CVE extends

---
 lib/python/bugs.py | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/lib/python/bugs.py b/lib/python/bugs.py
index 278e2ab79e8..5ec843b6218 100644
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -300,6 +300,28 @@ class Bug(BugBase):
             nts.append(notes[key])
         self.notes = nts
 
+class BugExtend(Bug):
+    def writeDB(self, cursor):
+        """Writes the record to an SQLite3 database."""
+
+        for (typ, c) in self.comments:
+            cursor.execute("""INSERT INTO bugs_notes
+            (bug_name, typ, comment) VALUES (?, ?, ?)""",
+                           (self.name, typ, c))
+
+        for n in self.notes:
+            n.writeDB(cursor, self.name)
+
+        import apsw
+        for x in self.xref:
+            try:
+                cursor.execute("""INSERT INTO bugs_xref
+                (source, target) VALUES (?, ?)""",
+                               (self.name, x))
+            except apsw.ConstraintError:
+                raise ValueError, \
+                      "cross reference to %s appears multiple times" % x
+
 class BugFromDB(Bug):
     def __init__(self, cursor, name):
         assert type(name) in types.StringTypes
@@ -441,6 +463,9 @@ class FileBase(debian_support.PackageFile):
         debian_support.PackageFile.__init__(self, name, fileObj)
         self.removed_packages = {}
 
+    def isExtend(self, name):
+        return False
+
     def isUniqueName(self, name):
         """Returns True if the name is a real, unique name."""
         return True
@@ -729,7 +754,11 @@ class FileBase(debian_support.PackageFile):
                         if first_bug:
                             break
                     record_name = temp_bug_name(first_bug, description)
-                yield self.finishBug(Bug(self.file.name, first_lineno, date,
+                if self.isExtend(record_name):
+                    cls = BugExtend
+                else:
+                    cls = Bug
+                yield self.finishBug(cls(self.file.name, first_lineno, date,
                                          record_name, description,
                                          comments, notes=pkg_notes, xref=xref))
 
@@ -774,6 +803,12 @@ class CVEFile(FileBase):
         bug.mergeNotes()
         return bug
 
+class CVECUSTOMERFile(CVEFile):
+    re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$')
+
+    def isExtend(self, name):
+        return True
+
 class DSAFile(FileBase):
     """A DSA file.
 
-- 
GitLab


From fafe483991149607160325eb6ef572faa141acf2 Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 11 May 2018 15:22:31 +0200
Subject: [PATCH 6/9] Simplify Extends support

BugExtend.writeDB() is pretty similar to BugBase's, so
update the latter to take extends into account when
necessary to avoid unneeded duplicated code.
---
 lib/python/bugs.py | 64 +++++++++++++++-------------------------------
 1 file changed, 20 insertions(+), 44 deletions(-)

diff --git a/lib/python/bugs.py b/lib/python/bugs.py
index 5ec843b6218..676f5f540f3 100644
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -201,6 +201,7 @@ class BugBase:
         self.notes = []
         self.xref = []
         self.not_for_us = False
+        self.is_extend = False
 
     def isFromCVE(self):
         """Returns True if the name has been officially assigned.
@@ -226,16 +227,18 @@ class BugBase:
             not_for_us = 0
 
         import apsw
-        try:
-            cursor.execute("""INSERT INTO bugs
-            (name, cve_status, not_for_us, description, release_date,
-             source_file, source_line)
-            VALUES (?, ?, ?, ?, ?, ?, ?)""",
-                           (self.name, self.cveStatus(), not_for_us,
-                            self.description, self.date or '',
-                            self.source_file, self.source_line))
-        except apsw.ConstraintError:
-            raise ValueError, "bug name %s is not unique" % self.name
+
+        if not self.is_extend:
+            try:
+                cursor.execute("""INSERT INTO bugs
+                (name, cve_status, not_for_us, description, release_date,
+                 source_file, source_line)
+                VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                               (self.name, self.cveStatus(), not_for_us,
+                                self.description, self.date or '',
+                                self.source_file, self.source_line))
+            except apsw.ConstraintError:
+                raise ValueError, "bug name %s is not unique" % self.name
 
         for (typ, c) in self.comments:
             cursor.execute("""INSERT INTO bugs_notes
@@ -258,7 +261,7 @@ class Bug(BugBase):
     """Class for bugs for which we have some data."""
 
     def __init__(self, fname, lineno, date, name, description, comments, notes,
-                 xref, not_for_us=False):
+                 xref, not_for_us=False, is_extend=False):
         for n in notes:
             assert isinstance(n, PackageNote) \
                    or isinstance(n, PackageNoteNoDSA)
@@ -269,6 +272,7 @@ class Bug(BugBase):
         self.notes = notes
         self.xref = xref
         self.not_for_us = not_for_us
+        self.is_extend = is_extend
 
     def mergeNotes(self):
         """Merge notes so that there is only one note for each
@@ -300,28 +304,6 @@ class Bug(BugBase):
             nts.append(notes[key])
         self.notes = nts
 
-class BugExtend(Bug):
-    def writeDB(self, cursor):
-        """Writes the record to an SQLite3 database."""
-
-        for (typ, c) in self.comments:
-            cursor.execute("""INSERT INTO bugs_notes
-            (bug_name, typ, comment) VALUES (?, ?, ?)""",
-                           (self.name, typ, c))
-
-        for n in self.notes:
-            n.writeDB(cursor, self.name)
-
-        import apsw
-        for x in self.xref:
-            try:
-                cursor.execute("""INSERT INTO bugs_xref
-                (source, target) VALUES (?, ?)""",
-                               (self.name, x))
-            except apsw.ConstraintError:
-                raise ValueError, \
-                      "cross reference to %s appears multiple times" % x
-
 class BugFromDB(Bug):
     def __init__(self, cursor, name):
         assert type(name) in types.StringTypes
@@ -458,14 +440,12 @@ class FileBase(debian_support.PackageFile):
     re_rejected = re.compile(r'^(?:NOTE:\s+rejected|REJECTED)\s*$')
     re_note = re.compile(r'^NOTE:\s+(.*)$')
     re_todo = re.compile(r'^TODO:\s+(.*)$')
+    is_extend = False
 
     def __init__(self, name, fileObj=None):
         debian_support.PackageFile.__init__(self, name, fileObj)
         self.removed_packages = {}
 
-    def isExtend(self, name):
-        return False
-
     def isUniqueName(self, name):
         """Returns True if the name is a real, unique name."""
         return True
@@ -754,13 +734,10 @@ class FileBase(debian_support.PackageFile):
                         if first_bug:
                             break
                     record_name = temp_bug_name(first_bug, description)
-                if self.isExtend(record_name):
-                    cls = BugExtend
-                else:
-                    cls = Bug
-                yield self.finishBug(cls(self.file.name, first_lineno, date,
+                yield self.finishBug(Bug(self.file.name, first_lineno, date,
                                          record_name, description,
-                                         comments, notes=pkg_notes, xref=xref))
+                                         comments, notes=pkg_notes, xref=xref,
+                                         is_extend=self.is_extend))
 
     def finishBug(self, bug):
         """Applies a transformation to the bug after it has been
@@ -806,8 +783,7 @@ class CVEFile(FileBase):
 class CVECUSTOMERFile(CVEFile):
     re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$')
 
-    def isExtend(self, name):
-        return True
+    is_extend = True
 
 class DSAFile(FileBase):
     """A DSA file.
-- 
GitLab


From bf1f037a3270e19cda4fb350124fc21f3964f6c1 Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 11 May 2018 16:18:06 +0200
Subject: [PATCH 7/9] gen-DSA: allow other gen-* links

---
 bin/gen-DSA | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/bin/gen-DSA b/bin/gen-DSA
index 0eb389d51cc..f2ba43e82bc 100755
--- a/bin/gen-DSA
+++ b/bin/gen-DSA
@@ -22,7 +22,7 @@ set -e
 
 IDMODE=DSA
 case "$(basename "$0")" in
-    *gen-D[LS]A)
+    *gen-*)
 	IDMODE=${0#*gen-}
     ;;
 esac
@@ -333,11 +333,7 @@ setvar DEBFULLNAME
 setvar SPACEDDEBFULLNAME
 setvar PACKAGE
 setvar CVE "$CVE_LIST"
-if [ "$IDMODE" = DSA ]; then
-    setvar DSAID "$DAID"
-else
-    setvar DLAID "$DAID"
-fi
+setvar ${IDMODE}ID "$DAID"
 setvar BUGNUM
 setvar OLDOLDSTABLE
 setvar OLDSTABLE
-- 
GitLab


From b59cbe46e12d468b4a41cf599ca12a2dcd024b92 Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 11 May 2018 17:51:01 +0200
Subject: [PATCH 8/9] Rename CVECUSTOMERFile to CVEExtendFile

---
 lib/python/bugs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/python/bugs.py b/lib/python/bugs.py
index 676f5f540f3..bcfa9b1ce57 100644
--- a/lib/python/bugs.py
+++ b/lib/python/bugs.py
@@ -780,7 +780,11 @@ class CVEFile(FileBase):
         bug.mergeNotes()
         return bug
 
-class CVECUSTOMERFile(CVEFile):
+class CVEExtendFile(CVEFile):
+    # This is an extend file. The main CVEFile can have a 'CVE-2018-XXXX' (sic)
+    # identifier, which will get converted to TEMP-* automatically. However to
+    # refer to that one from here, we need to use the TEMP-* identifier, so we
+    # allow those in the regex
     re_cve = re.compile(r'^(CVE-\d{4}-(?:\d{4,}|XXXX)|TEMP-\d+-\S+)\s+(.*?)\s*$')
 
     is_extend = True
-- 
GitLab


From a0c205800723e54d383c1cb54969e4da3922edfb Mon Sep 17 00:00:00 2001
From: Emilio Pozuelo Monfort <pochu@debian.org>
Date: Fri, 8 Jun 2018 09:46:29 +0200
Subject: [PATCH 9/9] Document CVE extends support

---
 doc/security-team.d.o/security_tracker | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/doc/security-team.d.o/security_tracker b/doc/security-team.d.o/security_tracker
index eeea313ca8a..cb91082bf7b 100644
--- a/doc/security-team.d.o/security_tracker
+++ b/doc/security-team.d.o/security_tracker
@@ -612,3 +612,23 @@ The following commands build the databases for stable and run a python local ser
     make serve
 
 The website is now available as `http://127.0.0.1:10605/tracker/`.
+
+Setting up an extended instance
+-------------------------------
+
+The security tracker supports extra sources of data, which can be used
+to override or extend the information in CVE/list, and to support your
+own announce lists. To do that, add a CVEExtendFile source to
+`data/config.json`. Entries in that file can add information to an
+existing CVE, e.g. to mark it as fixed or ignored, or to mark it as
+affecting additional source packages. For example:
+
+CVE-2018-11646
+	- webkitgtk <unfixed>
+CVE-2016-1000340
+	[wheezy] - bouncycastle <not-affected> (Vulnerable code introduced later)
+
+You can also add an announce list of type DSAFile to `data/config.json`,
+and then symlink `bin/gen-DSA` to e.g. `bin/gen-MySA` and use that to
+create new advisories under your namespace. For that you will need to
+add a `data/mysa-needed.txt` file and `doc/MYSA.template`.
-- 
GitLab