Merge lp:~jtv/launchpad/xpi-dtd-parser into lp:launchpad

[Jeroen T. Vermeulen]

Extract XPI DTD parser.

= DTD Parser =

This extracts the code for parsing an XPI DTD file into a file of its own. It's just cleaner for now, but it also looks like we'll want to build a full-blown importer around this parser later.

To test:
{{{
./bin/test -vvc -m lp.translations -t dtd
}}}

Jeroen

[Leonard Richardson (leonardr)]

+1

review: Approve

=== modified file 'lib/lp/translations/scripts/validate_translations_file.py'
--- lib/lp/translations/scripts/validate_translations_file.py	2010-01-06 06:04:54 +0000
+++ lib/lp/translations/scripts/validate_translations_file.py	2010-07-20 11:02:51 +0000
@@ -15,8 +15,9 @@
 
 from canonical.launchpad import scripts
 from lp.translations.utilities.gettext_po_parser import POParser
+from lp.translations.utilities.mozilla_dtd_parser import DtdFile
 from lp.translations.utilities.mozilla_xpi_importer import (
-    DtdFile, MozillaZipImportParser)
+    MozillaZipImportParser)
 from lp.translations.utilities.xpi_manifest import XpiManifest
 
 

=== modified file 'lib/lp/translations/utilities/gettext_po_importer.py'
--- lib/lp/translations/utilities/gettext_po_importer.py	2009-07-17 00:26:05 +0000
+++ lib/lp/translations/utilities/gettext_po_importer.py	2010-07-20 11:02:51 +0000
@@ -1,19 +1,19 @@
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
 
 __all__ = [
-    'GettextPOImporter'
+    'GettextPOImporter',
     ]
 
 from zope.component import getUtility
 from zope.interface import implements
 
+from lp.translations.interfaces.translationfileformat import (
+    TranslationFileFormat)
 from lp.translations.interfaces.translationimporter import (
     ITranslationFormatImporter)
-from lp.translations.interfaces.translationfileformat import (
-    TranslationFileFormat)
 from lp.translations.utilities.gettext_po_parser import (
     POParser, POHeader)
 from canonical.librarian.interfaces import ILibrarianClient

=== added file 'lib/lp/translations/utilities/mozilla_dtd_parser.py'
--- lib/lp/translations/utilities/mozilla_dtd_parser.py	1970-01-01 00:00:00 +0000
+++ lib/lp/translations/utilities/mozilla_dtd_parser.py	2010-07-20 11:02:51 +0000
@@ -0,0 +1,144 @@
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Importer for DTD files as found in XPI archives."""
+
+__metaclass__ = type
+__all__ = [
+    'DtdFile'
+    ]
+
+from old_xmlplus.parsers.xmlproc import dtdparser, xmldtd, utils
+
+from lp.translations.interfaces.translationimporter import (
+    TranslationFormatInvalidInputError,
+    TranslationFormatSyntaxError)
+from lp.translations.utilities.translation_common_format import (
+    TranslationMessageData)
+from lp.translations.interfaces.translations import TranslationConstants
+
+
+class MozillaDtdConsumer(xmldtd.WFCDTD):
+    """Mozilla DTD translatable message parser.
+
+    msgids are stored as entities. This class extracts it along
+    with translations, comments and source references.
+    """
+    def __init__(self, parser, filename, chrome_path, messages):
+        self.started = False
+        self.last_comment = None
+        self.chrome_path = chrome_path
+        self.messages = messages
+        self.filename = filename
+        xmldtd.WFCDTD.__init__(self, parser)
+
+    def dtd_start(self):
+        """See `xmldtd.WFCDTD`."""
+        self.started = True
+
+    def dtd_end(self):
+        """See `xmldtd.WFCDTD`."""
+        self.started = False
+
+    def handle_comment(self, contents):
+        """See `xmldtd.WFCDTD`."""
+        if not self.started:
+            return
+
+        if self.last_comment is not None:
+            self.last_comment += contents
+        elif len(contents) > 0:
+            self.last_comment = contents
+
+        if self.last_comment and not self.last_comment.endswith('\n'):
+            # Comments must end always with a new line.
+            self.last_comment += '\n'
+
+    def new_general_entity(self, name, value):
+        """See `xmldtd.WFCDTD`."""
+        if not self.started:
+            return
+
+        message = TranslationMessageData()
+        message.msgid_singular = name
+        # CarlosPerelloMarin 20070326: xmldtd parser does an inline
+        # parsing which means that the content is all in a single line so we
+        # don't have a way to show the line number with the source reference.
+        message.file_references_list = ["%s(%s)" % (self.filename, name)]
+        message.addTranslation(TranslationConstants.SINGULAR_FORM, value)
+        message.singular_text = value
+        message.context = self.chrome_path
+        message.source_comment = self.last_comment
+        self.messages.append(message)
+        self.started += 1
+        self.last_comment = None
+
+
+class DtdErrorHandler(utils.ErrorCounter):
+    """Error handler for the DTD parser."""
+    filename = None
+
+    def error(self, msg):
+        raise TranslationFormatSyntaxError(
+            filename=self.filename, message=msg)
+
+    def fatal(self, msg):
+        raise TranslationFormatInvalidInputError(
+            filename=self.filename, message=msg)
+
+
+class DummyDtdFile:
+    """"File" returned when DTD SYSTEM entity tries to include a file."""
+    done = False
+
+    def read(self, *args, **kwargs):
+        """Minimally satisfy attempt to read an included DTD file."""
+        if self.done:
+            return ''
+        else:
+            self.done = True
+            return '<!-- SYSTEM entities not supported. -->'
+
+    def close(self):
+        """Satisfy attempt to close file."""
+        pass
+
+
+class DtdInputSourceFactoryStub:
+    """Replace the class the DTD parser uses to include other DTD files."""
+
+    def create_input_source(self, sysid):
+        """Minimally satisfy attempt to open an included DTD file.
+
+        This is called when the DTD parser hits a SYSTEM entity.
+        """
+        return DummyDtdFile()
+
+
+class DtdFile:
+    """Class for reading translatable messages from a .dtd file.
+
+    It uses DTDParser which fills self.messages with parsed messages.
+    """
+    def __init__(self, filename, chrome_path, content):
+        self.messages = []
+        self.filename = filename
+        self.chrome_path = chrome_path
+
+        # .dtd files are supposed to be using UTF-8 encoding, if the file is
+        # using another encoding, it's against the standard so we reject it
+        try:
+            content = content.decode('utf-8')
+        except UnicodeDecodeError:
+            raise TranslationFormatInvalidInputError, (
+                'Content is not valid UTF-8 text')
+
+        error_handler = DtdErrorHandler()
+        error_handler.filename = filename
+
+        parser = dtdparser.DTDParser()
+        parser.set_error_handler(error_handler)
+        parser.set_inputsource_factory(DtdInputSourceFactoryStub())
+        dtd = MozillaDtdConsumer(parser, filename, chrome_path, self.messages)
+        parser.set_dtd_consumer(dtd)
+        parser.parse_string(content)

=== modified file 'lib/lp/translations/utilities/mozilla_xpi_importer.py'
--- lib/lp/translations/utilities/mozilla_xpi_importer.py	2010-01-05 13:44:13 +0000
+++ lib/lp/translations/utilities/mozilla_xpi_importer.py	2010-07-20 11:02:51 +0000
@@ -4,7 +4,6 @@
 __metaclass__ = type
 
 __all__ = [
-    'DtdFile',
     'MozillaXpiImporter',
     'MozillaZipImportParser',
     ]
@@ -12,8 +11,6 @@
 from cStringIO import StringIO
 import textwrap
 
-from old_xmlplus.parsers.xmlproc import dtdparser, xmldtd, utils
-
 from zope.component import getUtility
 from zope.interface import implements
 
@@ -27,13 +24,13 @@
 from lp.translations.utilities.translation_common_format import (
     TranslationFileData,
     TranslationMessageData)
+from lp.translations.utilities.mozilla_dtd_parser import DtdFile
 from lp.translations.utilities.mozilla_zip import (
     MozillaZipTraversal)
 from lp.translations.utilities.xpi_header import XpiHeader
 from canonical.librarian.interfaces import ILibrarianClient
 
 
-
 def add_source_comment(message, comment):
     """Add the given comment inside message.source_comment."""
     if message.source_comment:
@@ -160,130 +157,6 @@
             self.messages.append(message)
 
 
-class MozillaDtdConsumer(xmldtd.WFCDTD):
-    """Mozilla DTD translatable message parser.
-
-    msgids are stored as entities. This class extracts it along
-    with translations, comments and source references.
-    """
-    def __init__(self, parser, filename, chrome_path, messages):
-        self.started = False
-        self.last_comment = None
-        self.chrome_path = chrome_path
-        self.messages = messages
-        self.filename = filename
-        xmldtd.WFCDTD.__init__(self, parser)
-
-    def dtd_start(self):
-        """See `xmldtd.WFCDTD`."""
-        self.started = True
-
-    def dtd_end(self):
-        """See `xmldtd.WFCDTD`."""
-        self.started = False
-
-    def handle_comment(self, contents):
-        """See `xmldtd.WFCDTD`."""
-        if not self.started:
-            return
-
-        if self.last_comment is not None:
-            self.last_comment += contents
-        elif len(contents) > 0:
-            self.last_comment = contents
-
-        if self.last_comment and not self.last_comment.endswith('\n'):
-            # Comments must end always with a new line.
-            self.last_comment += '\n'
-
-    def new_general_entity(self, name, value):
-        """See `xmldtd.WFCDTD`."""
-        if not self.started:
-            return
-
-        message = TranslationMessageData()
-        message.msgid_singular = name
-        # CarlosPerelloMarin 20070326: xmldtd parser does an inline
-        # parsing which means that the content is all in a single line so we
-        # don't have a way to show the line number with the source reference.
-        message.file_references_list = ["%s(%s)" % (self.filename, name)]
-        message.addTranslation(TranslationConstants.SINGULAR_FORM, value)
-        message.singular_text = value
-        message.context = self.chrome_path
-        message.source_comment = self.last_comment
-        self.messages.append(message)
-        self.started += 1
-        self.last_comment = None
-
-
-class DtdErrorHandler(utils.ErrorCounter):
-    """Error handler for the DTD parser."""
-    filename = None
-
-    def error(self, msg):
-        raise TranslationFormatSyntaxError(
-            filename=self.filename, message=msg)
-
-    def fatal(self, msg):
-        raise TranslationFormatInvalidInputError(
-            filename=self.filename, message=msg)
-
-
-class DummyDtdFile:
-    """"File" returned when DTD SYSTEM entity tries to include a file."""
-    done = False
-
-    def read(self, *args, **kwargs):
-        """Minimally satisfy attempt to read an included DTD file."""
-        if self.done:
-            return ''
-        else:
-            self.done = True
-            return '<!-- SYSTEM entities not supported. -->'
-
-    def close(self):
-        """Satisfy attempt to close file."""
-        pass
-
-
-class DtdInputSourceFactoryStub:
-    """Replace the class the DTD parser uses to include other DTD files."""
-
-    def create_input_source(self, sysid):
-        """Minimally satisfy attempt to open an included DTD file.
-
-        This is called when the DTD parser hits a SYSTEM entity.
-        """
-        return DummyDtdFile()
-
-
-class DtdFile:
-    """Class for reading translatable messages from a .dtd file.
-
-    It uses DTDParser which fills self.messages with parsed messages.
-    """
-    def __init__(self, filename, chrome_path, content):
-        self.messages = []
-        self.filename = filename
-        self.chrome_path = chrome_path
-
-        # .dtd files are supposed to be using UTF-8 encoding, if the file is
-        # using another encoding, it's against the standard so we reject it
-        try:
-            content = content.decode('utf-8')
-        except UnicodeDecodeError:
-            raise TranslationFormatInvalidInputError, (
-                'Content is not valid UTF-8 text')
-
-        error_handler = DtdErrorHandler()
-        error_handler.filename = filename
-
-        parser = dtdparser.DTDParser()
-        parser.set_error_handler(error_handler)
-        parser.set_inputsource_factory(DtdInputSourceFactoryStub())
-        dtd = MozillaDtdConsumer(parser, filename, chrome_path, self.messages)
-        parser.set_dtd_consumer(dtd)
-        parser.parse_string(content)
 
 
 def valid_property_msgid(msgid):

=== modified file 'lib/lp/translations/utilities/tests/test_xpi_dtd_format.py'
--- lib/lp/translations/utilities/tests/test_xpi_dtd_format.py	2009-07-17 00:26:05 +0000
+++ lib/lp/translations/utilities/tests/test_xpi_dtd_format.py	2010-07-20 11:02:51 +0000
@@ -5,7 +5,7 @@
 
 import unittest
 
-from lp.translations.utilities.mozilla_xpi_importer import DtdFile
+from lp.translations.utilities.mozilla_dtd_parser import DtdFile
 from lp.translations.interfaces.translationimporter import (
     TranslationFormatInvalidInputError)