Makefile

@@ -112,14 +112,16 @@ update-lists:
 
 # Since October 16, 2015 the XML data feeds are no longer available for
 # download in an uncompressed format.
+# As per October 16, 2019, the XML data feeds were discontinued and NVD
+# only provides JSON feeds. Cf. https://bugs.debian.org/942670
 update-nvd:
 	mkdir -p data/nvd
 	for x in $$(seq 2002 $$(date +%Y)) ; do \
-	  name=nvdcve-$$x.xml.gz; \
-	  wget -q -Odata/nvd/$$name https://nvd.nist.gov/download/$$name || true; \
+	  name=nvdcve-1.1-$$x.json.gz; \
+	  wget -q -Odata/nvd/$$name https://nvd.nist.gov/feeds/json/cve/1.1/$$name || true; \
 	  gzip -f -d data/nvd/$$name || true; \
 	done
-	bin/update-nvd data/nvd/nvdcve-*.xml
+	bin/update-nvd data/nvd/nvdcve-*.json
 
 # Experimental code to compare the Debian and NVD CVE databases using
 # CPE values as common key.

lib/python/nvd.py

 # nvd.py -- simplistic NVD parser
 # Copyright (C) 2005 Florian Weimer <fw@deneb.enyo.de>
+# Copyright (C) 2019 Salvatore Bonaccorso <carnil@debian.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,68 +16,66 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-"""This module parses the XML files provided by the
+"""This module parses the JSON files provided by the
 National Vulnerability Database (NVD) <https://nvd.nist.gov/>
 """
 
-import xml.sax
-import xml.sax.handler
+import json
 
-class _Parser(xml.sax.handler.ContentHandler):
+class _Parser:
     """Parser helper class."""
 
     def __init__(self):
         self.result = []
-        self.start_dispatcher = {}
-        for x in ('entry', 'local', 'network', 'local_network', 'user_init',
-                  'avail', 'conf', 'int', 'sec_prot'):
-             self.start_dispatcher[x] = getattr(self, 'TAG_' + x)
-        self.path = []
 
-    def _noop(*args):
+    def parse(self, file):
+        cve_data=json.load(file)
+
+        for entry in cve_data['CVE_Items']:
+            # get CVE ID name
+            if 'cve' not in entry:
+                raise ValueError("No CVE entry present in CVE_Items")
+            if 'CVE_data_meta' not in entry['cve']:
+                raise ValueError("No CVE metadata entry present")
+            if 'ID' not in entry['cve']['CVE_data_meta']:
+                raise VAlueError("No CVE ID present for entry")
+            self.name=entry['cve']['CVE_data_meta']['ID']
+
+            # get CVE description
+            self.cve_desc=""
+            try:
+                self.cve_desc=entry['cve']['description']['description_data'][0].get('value')
+            except KeyError:
                 pass
 
-    def startElement(self, name, attrs):
-        self.path.append((name, attrs))
-        self.start_dispatcher.get(name, self._noop)(name, attrs)
+            # get discovered date
+            # TODO: re-implement or change database schema
+            self.discovered=""
 
-    def TAG_entry(self, name, attrs):
-        self.name = attrs['name'].encode('utf-8')
-        self.published = attrs['published'].encode('utf-8')
-        self.severity = attrs.get('severity', u'').encode('utf-8')
-        self.discovered = attrs.get('discovered', u'').encode('utf-8')
+            # get publication date
+            self.published=""
+            try:
+                self.published=entry.get('publishedDate')
+            except KeyError:
+                pass
 
-        self.cve_desc = ""
+            # get severity
+            self.severity=""
+            try:
+                self.severity=entry['impact']['baseMetricV2'].get('severity')
+            except KeyError:
+                pass
+
+            # initalize defaults
             self.range_local = self.range_remote = self.range_user_init = 0
 
             self.loss_avail = self.loss_conf = self.loss_int \
                 = self.loss_sec_prot_user = self.loss_sec_prot_admin \
                 = self.loss_sec_prot_other = 0
 
-    def TAG_local(self, name, attrs):
-        self.range_local = 1
-    def TAG_network(self, name, attrs):
-        self.range_remote = 1
-    def TAG_local_network(self, name, attrs):
-        self.range_remote = 1
-    def TAG_user_init(self, name, attrs):
-        self.range_user_init = 1
-    def TAG_avail(self, name, attrs):
-        self.loss_avail = 1
-    def TAG_conf(self, name, attrs):
-        self.loss_conf = 1
-    def TAG_int(self, name, attrs):
-        self.loss_int = 1
-    def TAG_sec_prot(self, name, attrs):
-        if 'user' in attrs:
-            self.loss_sec_prot_user = 1
-        if 'admin' in attrs:
-            self.loss_sec_prot_admin = 1
-        if 'other' in attrs:
-            self.loss_sec_prot_other = 1
-            
-    def endElement(self, name):
-        if name == 'entry':
+            # get range and loss values
+            # TODO: re-implement or change database schema
+
             self.result.append((self.name,
                                 self.cve_desc,
                                 self.discovered,
@@ -91,12 +90,6 @@ class _Parser(xml.sax.handler.ContentHandler):
                                 self.loss_sec_prot_user,
                                 self.loss_sec_prot_admin,
                                 self.loss_sec_prot_other))
-        del self.path[-1]
-
-    def characters(self, content):
-        (name, attrs) = self.path[-1]
-        if name == 'descript' and attrs['source'] == 'cve':
-            self.cve_desc += content
 
 def parse(file):
     """Parses the indicated file object.  Returns a list of tuples,
@@ -116,14 +109,12 @@ def parse(file):
     - security protection (admin) loss type flag
     - security protection (other) loss type flag
     """
-    parser = xml.sax.make_parser()
-    parser.setFeature(xml.sax.handler.feature_namespaces, 0)
+
     p = _Parser()
-    parser.setContentHandler(p)
-    parser.parse(file)
+    p.parse(file)
     return p.result
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     import sys
     for name in sys.argv[1:]:
         parse(open(name))