Merge ~alexmurray/ubuntu-cve-tracker:nvd-json into ubuntu-cve-tracker:master

[Alex Murray]

[Alex Murray (alexmurray)]

FYI - I plan to merge this on Wednesday if there is no feedback by then.

[Alex Murray (alexmurray)]

Approving as no feedback in nearly 2 weeks.

review: Approve

diff --git a/.gitignore b/.gitignore
index 766252e..ebc4960 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 .*.swp
 allitems.xml*
 nvdcve*.xml*
+nvdcve*.json*
 database.pickle*
 database-all.pickle*
 embargoed
diff --git a/scripts/check-cves b/scripts/check-cves
index 44e814b..bb77a33 100755
--- a/scripts/check-cves
+++ b/scripts/check-cves
@@ -15,6 +15,7 @@
 from __future__ import print_function
 
 import datetime
+import json
 import optparse
 import os
 import os.path
@@ -253,53 +254,40 @@ class PercentageFile(object):
         return self.f.close()
 
 
-def get_fake_nvd_xml_header():
-    '''Generate NVD header xml'''
-    today = "%s-%s-%s" % (datetime.date.today().year,
-                          datetime.date.today().month,
-                          datetime.date.today().day)
-    return '''<?xml version='1.0' encoding='UTF-8'?>
-<nvd xmlns="http://nvd.nist.gov/feeds/cve/1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" pub_date="%s" xsi:schemaLocation="http://nvd.nist.gov/feeds/cve/1.2 http://nvd.nist.gov/schema/nvdcve.xsd" nvd_xml_version="1.2">
-''' % (today)
-
-
-def get_fake_nvd_xml_footer():
-    '''Generate NVD footer xml'''
-    return '''</nvd>\n'''
-
-
-def convert_to_xml(cves, cve, desc):
-    s = '''  <entry type="CVE" severity="Medium" seq="%s" published="%s" name="%s" modified="%s" CVSS_version="2.0" CVSS_vector="(AV:N/AC:M/Au:N/C:N/I:P/A:P)" CVSS_score="0.0" CVSS_impact_subscore="0.0" CVSS_exploit_subscore="0.0" CVSS_base_score="0.0">
-    <desc>
-      <descript source="cve">%s</descript>
-    </desc>
-    <loss_types />
-    <range />
-''' % (re.sub(r'CVE\-', '', cve),
-       cves[cve]['date'].strftime("%Y-%m-%d"),
-       cve,
-       cves[cve]['date'].strftime("%Y-%m-%d"),
-       desc)
-
-    if cve in cves and 'refs' in cves[cve] and len(cves[cve]['refs']) > 0:
-        s += '    <refs>\n'
-        for r in cves[cve]['refs']:
-            s += '<ref url="%s" source="MISC">%s</ref>\n' % (r, r)
-        s += '    </refs>\n'
-    else:
-        s += '    <refs />\n'
+def convert_to_nvd(cves=[], desc=""):
+    # convert to nvd format dict (like nvd json)
+    nvd = {"CVE_data_type": "CVE",
+           "CVE_data_format": "MITRE",
+           "CVE_data_version": "4.0",
+           "CVE_data_timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
+           "CVE_Items": []}
 
-    s += '''    <vuln_soft />
-  </entry>
-'''
-    return s
+    keys = cves.keys()
+    keys.sort()
+    for cve in keys:
+        refs = []
+        if cve in cves and 'refs' in cves[cve] and len(cves[cve]['refs']) > 0:
+            for r in cves[cve]['refs']:
+                refs.append({"name": r,
+                             "url": r,
+                             "refsource": "MISC"})
+        nvd["CVE_Items"].append({"cve":
+                                 {"data_type": "CVE",
+                                  "data_format": "MITRE",
+                                  "data_version": "4.0",
+                                  "CVE_data_meta": {"ID": cve},
+                                  "references": {"reference_data": refs},
+                                  "description": {"description_data": [
+                                    {"lang": "en",
+                                     "value": desc(cve) if desc else ""}
+                                  ]}},
+                                 "publishedDate": cves[cve]['date'].strftime("%Y-%m-%dT%H:%M:%SZ")})
+    return nvd
 
 
 def import_debian(handler):
     '''Import Debian CVEs and DSAs missing from the tracker'''
     cves = dict()
-    cve = None
-    xml = get_fake_nvd_xml_header()
 
     today = datetime.date.today()
     known = set(CVEKnownList + CVEIgnoreList)
@@ -370,15 +358,9 @@ def import_debian(handler):
             if opt.verbose:
                 print("Processing %s: %s (%s)" % (cve, handler.debian[cve]['desc'], cves[cve]['date']), file=sys.stderr)
 
-    keys = cves.keys()
-    keys.sort()
-    for c in keys:
-        xml += convert_to_xml(cves, c, cves[c]['subject'])
-
-    xml += get_fake_nvd_xml_footer()
-
-    tmp, tmpname = tempfile.mkstemp(prefix='debian-import_')
-    os.write(tmp, xml)
+    nvd = convert_to_nvd(cves, lambda cve: cves[cve]['subject'])
+    tmp, tmpname = tempfile.mkstemp(prefix='debian-import_', suffix='.json')
+    os.write(tmp, json.dumps(nvd))
     os.close(tmp)
 
     return tmpname
@@ -386,14 +368,14 @@ def import_debian(handler):
 
 def read_locate_cves_output(f):
     '''Read in output of UCT/scripts/locate_cves.py
-       This is sneaky because we read in the output and then output a fake XML
+       This is sneaky because we read in the output and then output a fake JSON
        file for processing.
     '''
     if not os.path.isfile(f):
         print("'%s' not a file" % f, file=sys.stderr)
         sys.exit(1)
 
-    name = os.path.abspath(f + ".xml")
+    name = os.path.abspath(f + ".json")
     if os.path.exists(name):
         print("'%s' already exists" % name, file=sys.stderr)
         sys.exit(1)
@@ -459,18 +441,11 @@ def read_locate_cves_output(f):
             url = "http://www.openwall.com/lists/oss-security/%s" % (cves[cve]['date'].strftime("%Y/%m/%d"))
             cves[cve].setdefault('refs', [] + [url])
 
-    xml = get_fake_nvd_xml_header()
-
-    keys = cves.keys()
-    keys.sort()
-    for c in keys:
-        desc = '''ML-Date: %s, ML-Subject: %s''' % (cves[c]['date'], escape(cves[c]['subject']))
-        xml += convert_to_xml(cves, c, desc)
-
-    xml += get_fake_nvd_xml_footer()
+    nvd = convert_to_nvd(cves, lambda c: '''ML-Date: %s, ML-Subject: %s''' %
+                         (cves[c]['date'], escape(cves[c]['subject'])))
 
     tmp, tmpname = tempfile.mkstemp()
-    os.write(tmp, xml)
+    os.write(tmp, json.dumps(nvd))
 
     os.close(tmp)
     shutil.move(tmpname, name)
@@ -492,9 +467,9 @@ def read_mbox_file(f):
     os.write(tmp, out)
     os.close(tmp)
 
-    xml_file = read_locate_cves_output(tmpname)
+    json_file = read_locate_cves_output(tmpname)
     os.unlink(tmpname)
-    return xml_file
+    return json_file
 
 
 def dpkg_compare_versions(v1, op, v2):
@@ -551,7 +526,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
         self.debian = None
 
         # File-type detection
-        self.use_nvd = None
+
 
         # Load debian CVE states, if configured
         if 'secure_testing_path' in cve_lib.config:
@@ -573,43 +548,53 @@ class CVEHandler(xml.sax.handler.ContentHandler):
         print("---------------------------")
         print("%5d total CVEs triaged" % (self.num_added + self.num_ignored))
 
-    def startElement(self, name, attrs):
-        # NVD items
-        if name == "entry":
-            # Detect broken XML
-            if self.use_nvd is not None and not self.use_nvd:
-                raise KeyError("saw 'entry' in Mitre XML")
-            if self.use_nvd is None and opt.verbose:
-                print("Detected NVD XML schema", file=sys.stderr)
-            self.use_nvd = True
-            self.curr_cve = attrs['name']
-            self.curr_public = attrs['published']
+    def parse_json(self, fp):
+        template_nvd = {"CVE_data_type": "CVE",
+                        "CVE_data_format": "MITRE",
+                        "CVE_data_version": "4.0"}
+        nvd = json.load(fp)
+        # check for expected fields
+        for key in template_nvd.keys():
+            if key not in nvd or nvd[key] != template_nvd[key]:
+                raise KeyError("NVD JSON in '%s' seems invalid" % fp.name)
+        for item in nvd["CVE_Items"]:
+            template_cve = {"data_type": "CVE",
+                            "data_format": "MITRE",
+                            "data_version": "4.0"}
+            if "publishedDate" in item:
+                # convert from YYYY-MM-DDTHH:MM:SSZ to YYYY-MM-DD HH:MM:SS UTC
+                self.curr_public = item["publishedDate"].replace("T", " ").replace("Z", ":00 UTC")
+            else:
+                self.curr_public = None
+            cve = item["cve"]
+            for key in template_cve.keys():
+                if key not in cve or cve[key] != template_cve[key]:
+                    raise KeyError("NVD JSON in '%s' seems invalid" % fp.name)
+
+            metadata = cve["CVE_data_meta"]
+            self.curr_cve = metadata["ID"]
             self.curr_refs = []
-            self.curr_url = None
             self.curr_desc = None
-            self.curr_desc_ready = False
-        if name == "descript":
-            self.curr_chars_collect = True
-            self.curr_chars = ""
-            if attrs['source'] == 'cve':
-                self.curr_desc_ready = True
-        # Mitre items
+            for ref in cve["references"]["reference_data"]:
+                self.curr_refs += [(ref["refsource"], ref["name"], ref["url"])]
+            # find an english description
+            for desc in cve["description"]["description_data"]:
+                if desc["lang"] == "en":
+                    self.curr_desc = desc["value"]
+            self.handle_cve()
+
+
+    def startElement(self, name, attrs):
         if name == "item":
-            # Detect broken XML
-            if self.use_nvd is not None and self.use_nvd:
-                raise KeyError("saw 'item' in NVD XML")
-            if self.use_nvd is None and opt.verbose:
-                print("Detected Mitre XML schema", file=sys.stderr)
-            self.use_nvd = False
+            if opt.verbose:
+                print("Parsing Mitre XML schema", file=sys.stderr)
             self.curr_cve = attrs['name']
             self.curr_refs = []
             self.curr_url = None
             self.curr_desc = None
-        if name == "desc" and self.use_nvd is False:
+        if name == "desc":
             self.curr_chars_collect = True
             self.curr_chars = ""
-
-        # Common
         if name == "ref":
             self.curr_chars_collect = True
             self.curr_chars = ""
@@ -626,23 +611,15 @@ class CVEHandler(xml.sax.handler.ContentHandler):
 
     def endElement(self, name):
         self.curr_chars_collect = False
-        # NVD items
-        if name == "descript" and self.curr_desc_ready:
-            self.curr_desc = self.curr_chars.encode("ascii", "replace")
-            self.curr_desc_ready = False
-        if name == "entry":
-            self.handle_cve()
         # Mitre items
-        if name == "desc" and not self.use_nvd:
+        if name == "desc":
             self.curr_desc = self.curr_chars.encode("ascii", "replace")
         if name == "item":
             self.handle_cve()
-        # Common
         if name == "ref":
             self.curr_refs += [(self.curr_source, self.curr_chars.encode("ascii", "replace"), self.curr_url)]
 
     def handle_cve(self):
-
         # Skip CVEs we know about already
         if self.curr_cve in self.cve_ignore:
             return
@@ -1347,19 +1324,19 @@ if opt.test:
     unittest.TextTestRunner(verbosity=2).run(suite)
     sys.exit(0)
 
-untriaged_xml = ""
+untriaged_json = ""
 if opt.untriaged:
-    untriaged_xml = read_locate_cves_output(opt.untriaged)
-    args.append(untriaged_xml)
+    untriaged_json = read_locate_cves_output(opt.untriaged)
+    args.append(untriaged_json)
 
 if opt.mbox:
-    untriaged_xml = read_mbox_file(opt.mbox)
-    args.append(untriaged_xml)
+    untriaged_json = read_mbox_file(opt.mbox)
+    args.append(untriaged_json)
 
-debian_import_xml = ""
+debian_import_json = ""
 if opt.import_missing_debian and handler.debian is not None:
-    debian_import_xml = import_debian(handler)
-    args.append(debian_import_xml)
+    debian_import_json = import_debian(handler)
+    args.append(debian_import_json)
 
 if len(args) == 0:
     args.append("https://cve.mitre.org/cve/downloads/allitems.xml")
@@ -1371,18 +1348,21 @@ for uri in args:
     else:
         readable = PercentageFile(uri)
     try:
-        parser.parse(readable)
+        if uri.endswith("json"):
+            handler.parse_json(readable)
+        else:
+            parser.parse(readable)
     except xml.sax._exceptions.SAXParseException as e:
         print("\n\nWARNING: %s is malformed:\n%s" % (uri, e))
         print("Aborting", file=sys.stderr)
         sys.exit(1)
     print('')
 
-# Leaving our fake xml around is icky
-if os.path.exists(untriaged_xml):
-    os.unlink(untriaged_xml)
-if os.path.exists(debian_import_xml):
-    os.unlink(debian_import_xml)
+# Leaving our fake json around is icky
+if os.path.exists(untriaged_json):
+    os.unlink(untriaged_json)
+if os.path.exists(debian_import_json):
+    os.unlink(debian_import_json)
 
 if opt.refresh:
     for cve in sorted(CVEKnownList):
diff --git a/scripts/process_cves b/scripts/process_cves
index fc16316..752e255 100755
--- a/scripts/process_cves
+++ b/scripts/process_cves
@@ -29,8 +29,8 @@ download() {
     url="$1"
     # if http, then download it with wget, otherwise, try to rsync it
     if is_http_url "$url" ; then
-        # Force the gzipped XML to be downloaded
-        # Required for nvdcve-recent.xml as of 2015-10-16
+        # Force the gzipped JSON to be downloaded
+        # Required for nvdcve-1.1-recent.json as of 2015-10-16
         url="${url}.gz"
         file=$(basename "$url")
 
@@ -45,16 +45,16 @@ download() {
 }
 
 download_yearly_files() {
-    base_url="${nvd_loc:-https://nvd.nist.gov/download}/"
+    base_url="${nvd_loc:-https://nvd.nist.gov/feeds/json/cve/1.1}/"
     # if http, download each file sequentially, otherwise, use glob with rsync
     if is_http_url "$base_url" ; then
         current_year=$(date +%Y)
         for year in $(seq 2004 "$current_year"); do
-            download "${base_url}nvdcve-${year}.xml"
+            download "${base_url}nvdcve-1.1-${year}.json"
         done
     else
         # TODO: Make Y3K compliant
-        download "${base_url}nvdcve-2*.xml"
+        download "${base_url}nvdcve-1.1-2*.json"
     fi
 }
 
@@ -89,13 +89,13 @@ update_debian() {
 update_files() {
     download "${mitre_loc:-https://cve.mitre.org/data/downloads}/allitems.xml"
     download_yearly_files
-    download "https://nvd.nist.gov/download/nvdcve-recent.xml"
+    download "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-recent.json"
 }
 
 full_refresh() {
     # get PubDate
-    echo "update PubDate (check-cves --refresh nvdcve-*.xml) ..."
-    ./scripts/check-cves --refresh nvdcve-*.xml
+    echo "update PubDate (check-cves --refresh nvdcve-*.json) ..."
+    ./scripts/check-cves --refresh nvdcve-*.json
 
     # get authoritative descriptions
     echo "get authoritative descriptions (check-cves --refresh ./allitems.xml) ..."
@@ -216,8 +216,8 @@ case "$action" in
 
         # kernel_team_merge
         echo "Skipping merge from kernel team"
-        echo "check-cves nvdcve-*.xml"
-        ./scripts/check-cves nvdcve-*.xml
+        echo "check-cves nvdcve-*.json"
+        ./scripts/check-cves nvdcve-*.json
         process_missing_debian
 
         check_syntax
@@ -232,8 +232,8 @@ case "$action" in
         echo "Skipping merge from kernel team"
 
         # process new ones
-        echo "check-cves ./nvdcve-recent.xml"
-        ./scripts/check-cves ./nvdcve-recent.xml
+        echo "check-cves ./nvdcve-1.1-recent.json"
+        ./scripts/check-cves ./nvdcve-1.1-recent.json
         echo "check-cves ./allitems.xml"
         ./scripts/check-cves ./allitems.xml
 
@@ -282,8 +282,8 @@ case "$action" in
         update_files
         # kernel_team_merge
         echo "Skipping merge from kernel team"
-        echo "check-cves ./nvdcve-recent.xml"
-        ./scripts/check-cves ./nvdcve-recent.xml
+        echo "check-cves ./nvdcve-1.1-recent.json"
+        ./scripts/check-cves ./nvdcve-1.1-recent.json
         process_missing_debian
         check_syntax
         nag_about_mailing_lists