Merge ~iconstantin/ubuntu-cve-tracker:master into ubuntu-cve-tracker:master

[Ian Constantin]

Updated file handling to ensure files are closed after use.

There is a mix of how files are handled in cve_lib and at times the files are not being closed. The sample code below results in the following warnings:

     Warning (from warnings module):
       File "/home/ianc/git-pulls/ubuntu-cve-tracker/scripts/cve_lib.py", line 1832
         for line in codecs.open(cve, encoding="utf-8").readlines():
     ResourceWarning: unclosed file <_io.BufferedReader name='active/CVE-2020-25717'>

     Warning (from warnings module):
       File "/home/ianc/git-pulls/ubuntu-cve-tracker/scripts/cve_lib.py", line 1832
         for line in codecs.open(cve, encoding="utf-8").readlines():
     ResourceWarning: unclosed file <_io.BufferedReader name='active/CVE-2020-25718'>

     Warning (from warnings module):
       File "/home/ianc/git-pulls/ubuntu-cve-tracker/scripts/cve_lib.py", line 1832
         for line in codecs.open(cve, encoding="utf-8").readlines():
     ResourceWarning: unclosed file <_io.BufferedReader name='active/CVE-2020-25719'>

~~~
#!/usr/bin/python3

import optparse, os, sys, warnings

uct_path = "/home/ianc/git-pulls/ubuntu-cve-tracker"
sys.path.insert(0, uct_path + "/scripts") # Need $UCT/scripts in path to find cve_lib
os.chdir(uct_path) # cd'ing to $UCT before importing cve_lib (required)
import cve_lib

with warnings.catch_warnings():
    warnings.simplefilter('default')

    parser = optparse.OptionParser()
    opt, args = parser.parse_args()

    opt.pkgfamily = ""
    opt.debug= ""
    opt.packages = []

    cves = ["CVE-2020-25717", "CVE-2020-25718", "CVE-2020-25719"]
    uems = []

    cve_lib.load_table(cves, uems, opt)
~~~

[Alex Murray (alexmurray)]

LGTM! Thanks.

review: Approve

diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index a402f74..5d3275c 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -1302,7 +1302,9 @@ def read_config_file(config_file):
         # Dapper lacks this class, so reimplement it quickly
         class ConfigObj(dict):
             def __init__(self, filepath):
-                for line in open(filepath).readlines():
+                with open(filepath) as inF:
+                    lines = inF.readlines()
+                for line in lines:
                     line = line.strip()
                     if line.startswith('#') or len(line) == 0:
                         continue
@@ -1335,7 +1337,9 @@ def read_config():
 def drop_dup_release(cve, rel):
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
     saw = set()
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if line.startswith('%s_' % (rel)):
             pkg = line.split('_')[1].split(':')[0]
             if pkg not in saw:
@@ -1349,7 +1353,9 @@ def drop_dup_release(cve, rel):
 
 def clone_release(cve, pkg, oldrel, newrel):
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if line.startswith('%s_%s:' % (oldrel, pkg)):
             newline = line.replace('%s_%s:' % (oldrel, pkg), '%s_%s:' % (newrel, pkg), 1)
             output.write(newline)
@@ -1360,7 +1366,9 @@ def clone_release(cve, pkg, oldrel, newrel):
 
 def update_state(cve, pkg, rel, state, details):
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if line.startswith('%s_%s:' % (rel, pkg)):
             line = '%s_%s: %s' % (rel, pkg, state)
             if details:
@@ -1373,7 +1381,9 @@ def update_state(cve, pkg, rel, state, details):
 
 def add_state(cve, pkg, rel, state, details, after_rel):
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if line.startswith('%s_%s:' % (after_rel, pkg)):
             output.write(line)
             line = '%s_%s: %s' % (rel, pkg, state)
@@ -1396,7 +1406,9 @@ def prepend_field(cve, field, value):
 def update_field(cve, field, value=None):
     found = False
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if line.startswith('%s:' % (field)):
             found = True
             if value is None:
@@ -1419,7 +1431,9 @@ def drop_field(cve, field):
 def add_reference(cve, url):
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
     in_references = False
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if in_references and not line.startswith(' '):
             output.write(' ' + url + '\n')
             in_references = False
@@ -1452,7 +1466,8 @@ def add_cvss(cve, source, cvss):
     in_cvss = False
     found_cvss = False
     updated = False
-    lines = codecs.open(cve, encoding="utf-8").readlines()
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
     for line in lines:
         if not line.startswith('CVSS:') and not in_cvss:
             output.write(line)
@@ -1531,7 +1546,9 @@ def add_patch(cve, pkg, url, type="patch"):
     in_patch = False
 
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if in_patch and not line.startswith(' '):
             output.write(' ' + type + ': ' + url + '\n')
             in_patch = False
@@ -1558,7 +1575,9 @@ def update_multiline_field(cve, field, text):
         text = '\n' + text
     output = codecs.open(cve + ".new", 'w', encoding="utf-8")
     skip = 0
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         if skip and line.startswith(' '):
             continue
         skip = 0
@@ -1828,8 +1847,9 @@ def load_cve(cve, strict=False, srcmap=None):
     linenum = 0
     notes_parser = NotesParser()
     cvss_entries = []
-
-    for line in codecs.open(cve, encoding="utf-8").readlines():
+    with codecs.open(cve, encoding="utf-8") as inF:
+        lines = inF.readlines()
+    for line in lines:
         line = line.rstrip()
         linenum += 1
 
@@ -2343,6 +2363,8 @@ def load_debian_cves(filename, verbose=True):
         except:
             print("Error parsing line %d: '%s'" % (count, line), file=sys.stderr)
             raise
+
+    cvelist.close()
     return debian
 
 
@@ -2351,7 +2373,10 @@ def load_ignored_reasons(filename):
 
     ignored = dict()
 
-    for line in open(filename):
+    with open(filename) as inF:
+        lines = inF.readlines()
+
+    for line in lines:
         line = line.strip()
         if len(line) == 0 or line.startswith('#'):
             continue