Merge lp:~cjwatson/launchpad/cve-update-namespace into lp:launchpad

[Colin Watson]

Update CVE database XML namespace to match current file on cve.mitre.org.

It seems that cve.mitre.org recently made a slight change to the format of their allitems.xml.gz file (appending "/1.0" to the XML namespace), so update-cve is failing with OOPS-7f53d265afdaa4bfc02ea74330c001e6 and similar. This updates our parser to cope.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/bugs/doc/cve-update.txt'
--- lib/lp/bugs/doc/cve-update.txt	2011-12-29 05:29:36 +0000
+++ lib/lp/bugs/doc/cve-update.txt	2016-08-16 16:14:41 +0000
@@ -12,7 +12,12 @@
 and it does not attempt to connect to the outside world to verify that is
 the case.
 
-    >>> import subprocess, os.path, sys
+    >>> import gzip
+    >>> import os.path
+    >>> import shutil
+    >>> import subprocess
+    >>> import sys
+    >>> import tempfile
     >>> import transaction
     >>> from lp.services.config import config
 
@@ -24,18 +29,28 @@
     10
 
     >>> script = os.path.join(config.root, 'cronscripts', 'update-cve.py')
-    >>> url_base = os.path.join(
-    ...     config.root, 'lib', 'lp', 'bugs', 'tests', 'data'
-    ...     )
+    >>> base = os.path.join(config.root, 'lib', 'lp', 'bugs', 'tests', 'data')
 
 Now run the cronscript.
 
-    >>> url = 'file://%s' % os.path.join(url_base, 'cvedb_init.xml.gz')
-    >>> process = subprocess.Popen(
-    ...     [sys.executable, script, '-u', url], stdin=subprocess.PIPE,
-    ...     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-    ...     )
-    >>> (output, empty) = process.communicate()
+    >>> def update_from_file(path):
+    ...     tempdir = tempfile.mkdtemp()
+    ...     compressed_path = os.path.join(
+    ...         tempdir, os.path.basename(path) + '.gz')
+    ...     try:
+    ...         with open(path, 'rb') as uncompressed:
+    ...             with gzip.open(compressed_path, 'wb') as compressed:
+    ...                 shutil.copyfileobj(uncompressed, compressed)
+    ...         url = 'file://%s' % compressed_path
+    ...         process = subprocess.Popen(
+    ...             [sys.executable, script, '-u', url],
+    ...             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+    ...             stderr=subprocess.STDOUT)
+    ...         return process.communicate()
+    ...     finally:
+    ...         shutil.rmtree(tempdir)
+
+    >>> output, empty = update_from_file(os.path.join(base, 'cvedb_init.xml'))
     >>> print output
     INFO    Creating lockfile: /var/lock/launchpad-updatecve.lock
     ...
@@ -63,7 +78,7 @@
     ...
     <BLANKLINE>
 
-And lets make sure we got the right number of CVE entries.
+And let's make sure we got the right number of CVE entries.
 
     >>> transaction.commit()
     >>> print Cve.select().count()
@@ -84,12 +99,8 @@
 
 Now, let's run an import of the update db.
 
-    >>> url = 'file://%s' % os.path.join(url_base, 'cvedb_update.xml.gz')
-    >>> process = subprocess.Popen(
-    ...     [sys.executable, script, '-u', url], stdin=subprocess.PIPE,
-    ...     stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-    ...     )
-    >>> (output, empty) = process.communicate()
+    >>> output, empty = update_from_file(
+    ...     os.path.join(base, 'cvedb_update.xml'))
     >>> print output
     INFO    Creating lockfile: /var/lock/launchpad-updatecve.lock
     ...
@@ -129,7 +140,7 @@
     >>> print Cve.select().count()
     21
 
-And lets make sure the modification time of 2005-2734 was updated, as were
+And let's make sure the modification time of 2005-2734 was updated, as were
 the number of comments.
 
     >>> c.sync()

=== modified file 'lib/lp/bugs/scripts/cveimport.py'
--- lib/lp/bugs/scripts/cveimport.py	2015-07-08 16:05:11 +0000
+++ lib/lp/bugs/scripts/cveimport.py	2016-08-16 16:14:41 +0000
@@ -33,7 +33,7 @@
     )
 
 
-CVEDB_NS = '{http://cve.mitre.org/cve/downloads}'
+CVEDB_NS = '{http://cve.mitre.org/cve/downloads/1.0}'
 
 
 def getText(elem):

=== added file 'lib/lp/bugs/tests/data/cvedb_init.xml'
--- lib/lp/bugs/tests/data/cvedb_init.xml	1970-01-01 00:00:00 +0000
+++ lib/lp/bugs/tests/data/cvedb_init.xml	2016-08-16 16:14:41 +0000
@@ -0,0 +1,85 @@
+<?xml version="1.0"?>
+<cve xmlns="http://cve.mitre.org/cve/downloads/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+<item type="CVE" name="CVE-1999-0002" seq="1999-0002">
+<desc>Buffer overflow in NFS mountd gives root access to remote attackers, mostly in Linux systems.</desc>
+<refs>
+<ref source="SGI">19981006-01-I</ref>
+<ref source="BID">121</ref>
+<ref source="XF">linux-mountd-bo</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0003" seq="1999-0003">
+<desc>Execute commands as root via buffer overflow in Tooltalk database server (rpc.ttdbserverd)</desc>
+<refs>
+<ref source="CERT">CA-98.11.tooltalk</ref>
+<ref source="SGI">19981101-01-A</ref>
+<ref source="XF">aix-ttdbserver</ref>
+<ref source="XF">tooltalk</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0005" seq="1999-0005">
+<desc>Arbitrary command execution via IMAP buffer overflow in authenticate command.</desc>
+<refs>
+<ref source="CERT">CA-98.09.imapd</ref>
+<ref source="BID">130</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0007" seq="1999-0007">
+<desc>Information from encrypted sessions via PKCS #1</desc>
+<refs>
+<ref source="CERT">CA-98.07.PKCS</ref>
+<ref source="XF">nt-ssl-fix</ref>
+</refs>
+</item>
+
+<item type="CAN" name="CAN-2005-2761" seq="2005-2761">
+<phase date="20050831">Assigned</phase>
+<desc>Cross-site scripting (XSS) vulnerability in phpGroupWare 0.9.16.000 allows administrators to inject arbitrary web script or HTML by modifying the main screen message.</desc>
+<refs>
+<ref source="MISC" url="http://savannah.gnu.org/bugs/?func=detailitem&amp;item_id=13863">http://savannah.gnu.org/bugs/?func=detailitem&amp;item_id=13863</ref>
+<ref source="DEBIAN" url="http://www.debian.org/security/2005/dsa-798">DSA-798</ref>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2762" seq="2005-2762">
+<phase date="20050831">Assigned</phase>
+<desc>** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem.  When the candidate has been publicized, the details for this candidate will be provided.</desc>
+<refs>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2763" seq="2005-2763">
+<phase date="20050901">Assigned</phase>
+<desc>** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem.  When the candidate has been publicized, the details for this candidate will be provided.</desc>
+<refs>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2764" seq="2005-2764">
+<phase date="20050901">Assigned</phase>
+<desc>** RESERVED ** This candidate has been reserved by a company or individual that will use it when announcing a new security problem.  When the candidate has been publicized, the details for this candidate will be provided.</desc>
+<refs>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+</cve>

=== removed file 'lib/lp/bugs/tests/data/cvedb_init.xml.gz'
Binary files lib/lp/bugs/tests/data/cvedb_init.xml.gz	2011-12-21 18:40:25 +0000 and lib/lp/bugs/tests/data/cvedb_init.xml.gz	1970-01-01 00:00:00 +0000 differ
=== added file 'lib/lp/bugs/tests/data/cvedb_update.xml'
--- lib/lp/bugs/tests/data/cvedb_update.xml	1970-01-01 00:00:00 +0000
+++ lib/lp/bugs/tests/data/cvedb_update.xml	2016-08-16 16:14:41 +0000
@@ -0,0 +1,127 @@
+<?xml version="1.0"?>
+<cve xmlns="http://cve.mitre.org/cve/downloads/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<item type="CVE" name="CVE-1999-0002" seq="1999-0002">
+<desc>Buffer overflow in NFS mountd gives root access to remote attackers, mostly in Linux systems.</desc>
+<refs>
+<ref source="SGI">19981006-01-I</ref>
+<ref source="CERT">CA-98.12.mountd</ref>
+<ref source="CIAC">J-006</ref>
+<ref source="BID">121</ref>
+<ref source="XF">linux-mountd-bo</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0003" seq="1999-0003">
+<desc>Execute commands as root via buffer overflow in Tooltalk database server (rpc.ttdbserverd)</desc>
+<refs>
+<ref source="NAI">NAI-29</ref>
+<ref source="CERT">CA-98.11.tooltalk</ref>
+<ref source="SGI">19981101-01-A</ref>
+<ref source="SGI">19981101-01-PX</ref>
+<ref source="XF">aix-ttdbserver</ref>
+<ref source="XF">tooltalk</ref>
+<ref source="BID">122</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0005" seq="1999-0005">
+<desc>Arbitrary command execution via IMAP buffer overflow in authenticate command.</desc>
+<refs>
+<ref source="CERT">CA-98.09.imapd</ref>
+<ref source="SUN">00177</ref>
+<ref source="BID">130</ref>
+<ref source="XF">imap-authenticate-bo</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0006" seq="1999-0006">
+<desc>Buffer overflow in POP servers based on BSD/Qualcomm's qpopper allows remote attackers to gain root access using a long PASS command.</desc>
+<refs>
+<ref source="CERT">CA-98.08.qpopper_vul</ref>
+<ref source="SGI">19980801-01-I</ref>
+<ref source="AUSCERT">AA-98.01</ref>
+<ref source="XF">qpopper-pass-overflow</ref>
+<ref source="BID">133</ref>
+</refs>
+</item>
+
+<item type="CVE" name="CVE-1999-0007" seq="1999-0007">
+<desc>Information from SSL-encrypted sessions via PKCS #1</desc>
+<refs>
+<ref source="CERT">CA-98.07.PKCS</ref>
+<ref source="XF">nt-ssl-fix</ref>
+</refs>
+</item>
+
+<item type="CAN" name="CAN-2005-2761" seq="2005-2761">
+<phase date="20050831">Assigned</phase>
+<desc>Cross-site scripting (XSS) vulnerability in phpGroupWare 0.9.16.000 allows administrators to inject arbitrary web script or HTML by modifying the main screen message.</desc>
+<refs>
+<ref source="MISC" url="http://savannah.gnu.org/bugs/?func=detailitem&amp;item_id=13863">http://savannah.gnu.org/bugs/?func=detailitem&amp;item_id=13863</ref>
+<ref source="DEBIAN" url="http://www.debian.org/security/2005/dsa-798">DSA-798</ref>
+<ref source="SECUNIA" url="http://secunia.com/advisories/16558">16558</ref>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2734" seq="2005-2734">
+<phase date="20050831">Assigned</phase>
+<desc>** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem.  When the candidate has been publicized, the details for this candidate will be provided.</desc>
+<refs>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2763" seq="2005-2763">
+<phase date="20050901">Assigned</phase>
+<desc>** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem.  When the candidate has been publicized, the details for this candidate will be provided.</desc>
+<refs>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2764" seq="2005-2764">
+<phase date="20050901">Assigned</phase>
+<desc>** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem.  When the candidate has been publicized, the details for this candidate will be provided.</desc>
+<refs>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2765" seq="2005-2765">
+<phase date="20050901">Assigned</phase>
+<desc>The user interface in the Windows Firewall does not properly display certain malformed entries in the Windows Registry, which makes it easier for attackers with administrator privileges to hide activities if the administrator only uses the Windows Firewall interface to monitor exceptions.  NOTE: the vendor disputes this issue, saying that since administrative privileges are already required, it is not a vulnerability.  CVE has not yet formally decided if such &quot;information hiding&quot; issues should be included.</desc>
+<refs>
+<ref source="MSKB" url="http://www.microsoft.com/technet/security/advisory/897663.mspx">897663</ref>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+<item type="CAN" name="CAN-2005-2766" seq="2005-2766">
+<phase date="20050902">Assigned</phase>
+<desc>Symantec AntiVirus Corporate Edition 9.0.1.x and 9.0.4.x, and possibly other versions, when obtaining updates from an internal LiveUpdate server, stored sensitive information in cleartext in the Log.Liveupdate log file, which allows attackers to obtain the username and password to the internal LiveUpdate server.</desc>
+<refs>
+<ref source="BUGTRAQ" url="http://marc.theaimsgroup.com/?l=bugtraq&amp;m=112552401413998&amp;w=2">20050831 Vulnerability in Symantec Anti Virus Corporate Edition v9.x</ref>
+</refs>
+<votes>
+</votes>
+<comments>
+</comments>
+</item>
+
+</cve>

=== removed file 'lib/lp/bugs/tests/data/cvedb_update.xml.gz'
Binary files lib/lp/bugs/tests/data/cvedb_update.xml.gz	2011-12-21 18:40:25 +0000 and lib/lp/bugs/tests/data/cvedb_update.xml.gz	1970-01-01 00:00:00 +0000 differ