Merge lp:~cjwatson/launchpad/ttb-from-disk into lp:launchpad
- ttb-from-disk
- Merge into devel
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 18362 |
Proposed branch: | lp:~cjwatson/launchpad/ttb-from-disk |
Merge into: | lp:launchpad |
Diff against target: |
541 lines (+94/-78) 16 files modified
lib/lp/translations/doc/poexport-language-pack.txt (+3/-3) lib/lp/translations/interfaces/translationimporter.py (+7/-6) lib/lp/translations/interfaces/translationimportqueue.py (+5/-3) lib/lp/translations/model/translationimportqueue.py (+24/-14) lib/lp/translations/model/translationtemplatesbuildbehaviour.py (+7/-16) lib/lp/translations/scripts/upload_translations.py (+6/-6) lib/lp/translations/tests/test_translationimportqueue.py (+5/-5) lib/lp/translations/utilities/kde_po_importer.py (+12/-6) lib/lp/translations/utilities/tests/helpers.py (+3/-2) lib/lp/translations/utilities/tests/test_gettext_po_importer.py (+3/-2) lib/lp/translations/utilities/tests/test_kde_po_importer.py (+3/-2) lib/lp/translations/utilities/tests/test_mozilla_xpi_importer.py (+3/-2) lib/lp/translations/utilities/tests/test_translation_importer.py (+6/-4) lib/lp/translations/utilities/tests/test_xpi_import.py (+3/-3) lib/lp/translations/utilities/tests/test_xpi_po_exporter.py (+2/-2) lib/lp/translations/utilities/tests/test_xpi_search.py (+2/-2) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad/ttb-from-disk |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+322759@code.launchpad.net |
Commit message
Allow TranslationImpo
Description of the change
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/translations/doc/poexport-language-pack.txt' | |||
2 | --- lib/lp/translations/doc/poexport-language-pack.txt 2016-02-05 16:51:12 +0000 | |||
3 | +++ lib/lp/translations/doc/poexport-language-pack.txt 2017-04-24 14:03:06 +0000 | |||
4 | @@ -153,11 +153,11 @@ | |||
5 | 153 | 153 | ||
6 | 154 | Attach the en-US.xpi (the template) file. | 154 | Attach the en-US.xpi (the template) file. |
7 | 155 | 155 | ||
9 | 156 | >>> en_US_xpi = get_en_US_xpi_file_to_import('en-US') | 156 | >>> en_US_xpi = get_en_US_xpi_file_to_import('en-US') |
10 | 157 | >>> translation_import_queue = getUtility(ITranslationImportQueue) | 157 | >>> translation_import_queue = getUtility(ITranslationImportQueue) |
11 | 158 | >>> by_maintainer = True | 158 | >>> by_maintainer = True |
12 | 159 | >>> template_entry = translation_import_queue.addOrUpdateEntry( | 159 | >>> template_entry = translation_import_queue.addOrUpdateEntry( |
14 | 160 | ... firefox_template.path, en_US_xpi.read(), by_maintainer, | 160 | ... firefox_template.path, en_US_xpi, by_maintainer, |
15 | 161 | ... mark, distroseries=series, sourcepackagename=spn, | 161 | ... mark, distroseries=series, sourcepackagename=spn, |
16 | 162 | ... potemplate=firefox_template) | 162 | ... potemplate=firefox_template) |
17 | 163 | 163 | ||
18 | @@ -166,7 +166,7 @@ | |||
19 | 166 | >>> es_xpi = get_en_US_xpi_file_to_import('en-US') | 166 | >>> es_xpi = get_en_US_xpi_file_to_import('en-US') |
20 | 167 | >>> firefox_es_translation = firefox_template.newPOFile('es') | 167 | >>> firefox_es_translation = firefox_template.newPOFile('es') |
21 | 168 | >>> translation_entry = translation_import_queue.addOrUpdateEntry( | 168 | >>> translation_entry = translation_import_queue.addOrUpdateEntry( |
23 | 169 | ... 'es.xpi', es_xpi.read(), by_maintainer, | 169 | ... 'es.xpi', es_xpi, by_maintainer, |
24 | 170 | ... mark, distroseries=series, sourcepackagename=spn, | 170 | ... mark, distroseries=series, sourcepackagename=spn, |
25 | 171 | ... potemplate=firefox_template, | 171 | ... potemplate=firefox_template, |
26 | 172 | ... pofile=firefox_es_translation) | 172 | ... pofile=firefox_es_translation) |
27 | 173 | 173 | ||
28 | === modified file 'lib/lp/translations/interfaces/translationimporter.py' | |||
29 | --- lib/lp/translations/interfaces/translationimporter.py 2013-01-07 02:40:55 +0000 | |||
30 | +++ lib/lp/translations/interfaces/translationimporter.py 2017-04-24 14:03:06 +0000 | |||
31 | @@ -1,4 +1,4 @@ | |||
33 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
34 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
35 | 3 | 3 | ||
36 | 4 | """Interfaces to handle translation files imports.""" | 4 | """Interfaces to handle translation files imports.""" |
37 | @@ -141,7 +141,7 @@ | |||
38 | 141 | """Return the translation file format for the given file extension. | 141 | """Return the translation file format for the given file extension. |
39 | 142 | 142 | ||
40 | 143 | :param file_extension: File extension including the dot. | 143 | :param file_extension: File extension including the dot. |
42 | 144 | :param file_contents: File contents. | 144 | :param file_contents: File contents (a seekable file object). |
43 | 145 | :return: A `TranslationFileFormat` for the given file extension | 145 | :return: A `TranslationFileFormat` for the given file extension |
44 | 146 | and file contents or None if it's not supported format. | 146 | and file contents or None if it's not supported format. |
45 | 147 | """ | 147 | """ |
46 | @@ -182,10 +182,11 @@ | |||
47 | 182 | def getFormat(file_contents): | 182 | def getFormat(file_contents): |
48 | 183 | """The file format of the import. | 183 | """The file format of the import. |
49 | 184 | 184 | ||
54 | 185 | :param file_contents: A unicode string with the contents of the file | 185 | :param file_contents: A seekable file object with the contents of |
55 | 186 | being imported. A returned format may sometimes be different | 186 | the file being imported. A returned format may sometimes be |
56 | 187 | from the base format of the `ITranslationFormatImporter`, and | 187 | different from the base format of the |
57 | 188 | that is determined based on the `contents`. | 188 | `ITranslationFormatImporter`, and that is determined based on |
58 | 189 | the `contents`. | ||
59 | 189 | :return: A `TranslationFileFormat` value. | 190 | :return: A `TranslationFileFormat` value. |
60 | 190 | """ | 191 | """ |
61 | 191 | 192 | ||
62 | 192 | 193 | ||
63 | === modified file 'lib/lp/translations/interfaces/translationimportqueue.py' | |||
64 | --- lib/lp/translations/interfaces/translationimportqueue.py 2014-08-20 06:59:52 +0000 | |||
65 | +++ lib/lp/translations/interfaces/translationimportqueue.py 2017-04-24 14:03:06 +0000 | |||
66 | @@ -1,4 +1,4 @@ | |||
68 | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
69 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
70 | 3 | 3 | ||
71 | 4 | from datetime import timedelta | 4 | from datetime import timedelta |
72 | @@ -313,7 +313,8 @@ | |||
73 | 313 | """Return a new or updated entry of the import queue. | 313 | """Return a new or updated entry of the import queue. |
74 | 314 | 314 | ||
75 | 315 | :arg path: is the path, with the filename, of the uploaded file. | 315 | :arg path: is the path, with the filename, of the uploaded file. |
77 | 316 | :arg content: is the file content. | 316 | :arg content: is the file content, or a seekable file object open on |
78 | 317 | the file content. | ||
79 | 317 | :arg by_maintainer: indicates if the file was uploaded by the | 318 | :arg by_maintainer: indicates if the file was uploaded by the |
80 | 318 | maintainer of the project or package. | 319 | maintainer of the project or package. |
81 | 319 | :arg importer: is the person that did the import. | 320 | :arg importer: is the person that did the import. |
82 | @@ -336,7 +337,8 @@ | |||
83 | 336 | only_templates=False): | 337 | only_templates=False): |
84 | 337 | """Add all .po or .pot files from the tarball at :content:. | 338 | """Add all .po or .pot files from the tarball at :content:. |
85 | 338 | 339 | ||
87 | 339 | :arg content: is a tarball stream. | 340 | :arg content: is the tarball content, or a seekable file object open |
88 | 341 | on the tarball. | ||
89 | 340 | :arg by_maintainer: indicates if the file was uploaded by the | 342 | :arg by_maintainer: indicates if the file was uploaded by the |
90 | 341 | maintainer of the project or package. | 343 | maintainer of the project or package. |
91 | 342 | :arg importer: is the person that did the import. | 344 | :arg importer: is the person that did the import. |
92 | 343 | 345 | ||
93 | === modified file 'lib/lp/translations/model/translationimportqueue.py' | |||
94 | --- lib/lp/translations/model/translationimportqueue.py 2015-07-08 16:05:11 +0000 | |||
95 | +++ lib/lp/translations/model/translationimportqueue.py 2017-04-24 14:03:06 +0000 | |||
96 | @@ -1,4 +1,4 @@ | |||
98 | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
99 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
100 | 3 | 3 | ||
101 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
102 | @@ -8,8 +8,8 @@ | |||
103 | 8 | 'TranslationImportQueue', | 8 | 'TranslationImportQueue', |
104 | 9 | ] | 9 | ] |
105 | 10 | 10 | ||
106 | 11 | from cStringIO import StringIO | ||
107 | 12 | import datetime | 11 | import datetime |
108 | 12 | from io import BytesIO | ||
109 | 13 | import logging | 13 | import logging |
110 | 14 | from operator import attrgetter | 14 | from operator import attrgetter |
111 | 15 | import os.path | 15 | import os.path |
112 | @@ -1030,6 +1030,19 @@ | |||
113 | 1030 | return ( | 1030 | return ( |
114 | 1031 | format, translation_importer.getTranslationFormatImporter(format)) | 1031 | format, translation_importer.getTranslationFormatImporter(format)) |
115 | 1032 | 1032 | ||
116 | 1033 | def _getFileObjectAndSize(self, file_or_data): | ||
117 | 1034 | """Get the size of a seekable file object.""" | ||
118 | 1035 | if (isinstance(file_or_data, basestring)): | ||
119 | 1036 | file_obj = BytesIO(file_or_data) | ||
120 | 1037 | file_size = len(file_or_data) | ||
121 | 1038 | else: | ||
122 | 1039 | file_obj = file_or_data | ||
123 | 1040 | start = file_obj.tell() | ||
124 | 1041 | file_obj.seek(0, os.SEEK_END) | ||
125 | 1042 | file_size = file_obj.tell() | ||
126 | 1043 | file_obj.seek(start) | ||
127 | 1044 | return file_obj, file_size | ||
128 | 1045 | |||
129 | 1033 | def addOrUpdateEntry(self, path, content, by_maintainer, importer, | 1046 | def addOrUpdateEntry(self, path, content, by_maintainer, importer, |
130 | 1034 | sourcepackagename=None, distroseries=None, | 1047 | sourcepackagename=None, distroseries=None, |
131 | 1035 | productseries=None, potemplate=None, pofile=None, | 1048 | productseries=None, potemplate=None, pofile=None, |
132 | @@ -1040,16 +1053,18 @@ | |||
133 | 1040 | "This one has either neither or both.") | 1053 | "This one has either neither or both.") |
134 | 1041 | assert productseries is None or sourcepackagename is None, ( | 1054 | assert productseries is None or sourcepackagename is None, ( |
135 | 1042 | "Can't upload to a sourcepackagename in a productseries.") | 1055 | "Can't upload to a sourcepackagename in a productseries.") |
137 | 1043 | assert content is not None and content != '', "Upload has no content." | 1056 | assert content is not None, "Upload has no content." |
138 | 1044 | assert path is not None and path != '', "Upload has no path." | 1057 | assert path is not None and path != '', "Upload has no path." |
139 | 1045 | 1058 | ||
140 | 1059 | file, size = self._getFileObjectAndSize(content) | ||
141 | 1060 | assert size is not None and size != 0, "Upload has empty content." | ||
142 | 1061 | |||
143 | 1046 | filename = os.path.basename(path) | 1062 | filename = os.path.basename(path) |
144 | 1047 | format, format_importer = self._getFormatAndImporter( | 1063 | format, format_importer = self._getFormatAndImporter( |
146 | 1048 | filename, content, format=format) | 1064 | filename, file, format=format) |
147 | 1065 | file.seek(0) | ||
148 | 1049 | 1066 | ||
149 | 1050 | # Upload the file into librarian. | 1067 | # Upload the file into librarian. |
150 | 1051 | size = len(content) | ||
151 | 1052 | file = StringIO(content) | ||
152 | 1053 | client = getUtility(ILibrarianClient) | 1068 | client = getUtility(ILibrarianClient) |
153 | 1054 | alias = client.addFile( | 1069 | alias = client.addFile( |
154 | 1055 | name=filename, size=size, file=file, | 1070 | name=filename, size=size, file=file, |
155 | @@ -1144,9 +1159,9 @@ | |||
156 | 1144 | num_files = 0 | 1159 | num_files = 0 |
157 | 1145 | conflict_files = [] | 1160 | conflict_files = [] |
158 | 1146 | 1161 | ||
160 | 1147 | tarball_io = StringIO(content) | 1162 | tarball_io, _ = self._getFileObjectAndSize(content) |
161 | 1148 | try: | 1163 | try: |
163 | 1149 | tarball = tarfile.open('', 'r|*', tarball_io) | 1164 | tarball = tarfile.open('', 'r:*', tarball_io) |
164 | 1150 | except (tarfile.CompressionError, tarfile.ReadError): | 1165 | except (tarfile.CompressionError, tarfile.ReadError): |
165 | 1151 | # If something went wrong with the tarfile, assume it's | 1166 | # If something went wrong with the tarfile, assume it's |
166 | 1152 | # busted and let the user deal with it. | 1167 | # busted and let the user deal with it. |
167 | @@ -1158,7 +1173,6 @@ | |||
168 | 1158 | path = self._makePath(name, filename_filter) | 1173 | path = self._makePath(name, filename_filter) |
169 | 1159 | if self._isTranslationFile(path, only_templates): | 1174 | if self._isTranslationFile(path, only_templates): |
170 | 1160 | upload_files[name] = path | 1175 | upload_files[name] = path |
171 | 1161 | tarball.close() | ||
172 | 1162 | 1176 | ||
173 | 1163 | if approver_factory is None: | 1177 | if approver_factory is None: |
174 | 1164 | approver_factory = TranslationNullApprover | 1178 | approver_factory = TranslationNullApprover |
175 | @@ -1167,14 +1181,10 @@ | |||
176 | 1167 | productseries=productseries, | 1181 | productseries=productseries, |
177 | 1168 | distroseries=distroseries, sourcepackagename=sourcepackagename) | 1182 | distroseries=distroseries, sourcepackagename=sourcepackagename) |
178 | 1169 | 1183 | ||
179 | 1170 | # Re-opening because we are using sequential access ("r|*") which is | ||
180 | 1171 | # so much faster. | ||
181 | 1172 | tarball_io.seek(0) | ||
182 | 1173 | tarball = tarfile.open('', 'r|*', tarball_io) | ||
183 | 1174 | for tarinfo in tarball: | 1184 | for tarinfo in tarball: |
184 | 1175 | if tarinfo.name not in upload_files: | 1185 | if tarinfo.name not in upload_files: |
185 | 1176 | continue | 1186 | continue |
187 | 1177 | file_content = tarball.extractfile(tarinfo).read() | 1187 | file_content = tarball.extractfile(tarinfo) |
188 | 1178 | 1188 | ||
189 | 1179 | path = upload_files[tarinfo.name] | 1189 | path = upload_files[tarinfo.name] |
190 | 1180 | entry = approver.approve(self.addOrUpdateEntry( | 1190 | entry = approver.approve(self.addOrUpdateEntry( |
191 | 1181 | 1191 | ||
192 | === modified file 'lib/lp/translations/model/translationtemplatesbuildbehaviour.py' | |||
193 | --- lib/lp/translations/model/translationtemplatesbuildbehaviour.py 2016-03-31 14:58:46 +0000 | |||
194 | +++ lib/lp/translations/model/translationtemplatesbuildbehaviour.py 2017-04-24 14:03:06 +0000 | |||
195 | @@ -1,4 +1,4 @@ | |||
197 | 1 | # Copyright 2010-2014 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2010-2017 Canonical Ltd. This software is licensed under the |
198 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
199 | 3 | 3 | ||
200 | 4 | """An `IBuildFarmJobBehaviour` for `TranslationTemplatesBuild`. | 4 | """An `IBuildFarmJobBehaviour` for `TranslationTemplatesBuild`. |
201 | @@ -110,26 +110,17 @@ | |||
202 | 110 | filename = yield self._readTarball( | 110 | filename = yield self._readTarball( |
203 | 111 | self.build.buildqueue_record, filemap, logger) | 111 | self.build.buildqueue_record, filemap, logger) |
204 | 112 | 112 | ||
205 | 113 | # XXX 2010-11-12 bug=674575 | ||
206 | 114 | # Please make addOrUpdateEntriesFromTarball() take files on | ||
207 | 115 | # disk; reading arbitrarily sized files into memory is | ||
208 | 116 | # dangerous. | ||
209 | 117 | if filename is None: | 113 | if filename is None: |
210 | 118 | logger.error("Build produced no tarball.") | 114 | logger.error("Build produced no tarball.") |
211 | 119 | else: | 115 | else: |
212 | 120 | tarball_file = open(filename) | 116 | tarball_file = open(filename) |
213 | 121 | try: | 117 | try: |
225 | 122 | tarball = tarball_file.read() | 118 | logger.debug("Uploading translation templates tarball.") |
226 | 123 | if tarball is None: | 119 | self._uploadTarball( |
227 | 124 | logger.error("Build produced empty tarball.") | 120 | self.build.buildqueue_record.specific_build.branch, |
228 | 125 | else: | 121 | tarball_file, logger) |
229 | 126 | logger.debug( | 122 | transaction.commit() |
230 | 127 | "Uploading translation templates tarball.") | 123 | logger.debug("Upload complete.") |
220 | 128 | self._uploadTarball( | ||
221 | 129 | self.build.buildqueue_record.specific_build.branch, | ||
222 | 130 | tarball, logger) | ||
223 | 131 | transaction.commit() | ||
224 | 132 | logger.debug("Upload complete.") | ||
231 | 133 | finally: | 124 | finally: |
232 | 134 | tarball_file.close() | 125 | tarball_file.close() |
233 | 135 | os.remove(filename) | 126 | os.remove(filename) |
234 | 136 | 127 | ||
235 | === modified file 'lib/lp/translations/scripts/upload_translations.py' | |||
236 | --- lib/lp/translations/scripts/upload_translations.py 2015-10-14 16:23:18 +0000 | |||
237 | +++ lib/lp/translations/scripts/upload_translations.py 2017-04-24 14:03:06 +0000 | |||
238 | @@ -1,4 +1,4 @@ | |||
240 | 1 | # Copyright 2011 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2011-2017 Canonical Ltd. This software is licensed under the |
241 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
242 | 3 | 3 | ||
243 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
244 | @@ -55,11 +55,11 @@ | |||
245 | 55 | raise LaunchpadScriptFailure( | 55 | raise LaunchpadScriptFailure( |
246 | 56 | "File not readable: %s" % filename) | 56 | "File not readable: %s" % filename) |
247 | 57 | self.logger.info("Uploading: %s." % filename) | 57 | self.logger.info("Uploading: %s." % filename) |
253 | 58 | content = open(filename).read() | 58 | with open(filename) as content: |
254 | 59 | queue.addOrUpdateEntry( | 59 | queue.addOrUpdateEntry( |
255 | 60 | filename, content, True, rosetta_team, | 60 | filename, content, True, rosetta_team, |
256 | 61 | sourcepackagename=self.sourcepackagename, | 61 | sourcepackagename=self.sourcepackagename, |
257 | 62 | distroseries=self.distroseries) | 62 | distroseries=self.distroseries) |
258 | 63 | self._commit() | 63 | self._commit() |
259 | 64 | 64 | ||
260 | 65 | self.logger.info("Done.") | 65 | self.logger.info("Done.") |
261 | 66 | 66 | ||
262 | === modified file 'lib/lp/translations/tests/test_translationimportqueue.py' | |||
263 | --- lib/lp/translations/tests/test_translationimportqueue.py 2013-07-26 12:48:37 +0000 | |||
264 | +++ lib/lp/translations/tests/test_translationimportqueue.py 2017-04-24 14:03:06 +0000 | |||
265 | @@ -1,4 +1,4 @@ | |||
267 | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
268 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
269 | 3 | 3 | ||
270 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
271 | @@ -409,7 +409,7 @@ | |||
272 | 409 | self._makeFile('po'), | 409 | self._makeFile('po'), |
273 | 410 | self._makeFile('xpi'), | 410 | self._makeFile('xpi'), |
274 | 411 | )) | 411 | )) |
276 | 412 | tarfile_content = LaunchpadWriteTarFile.files_to_string(files) | 412 | tarfile_content = LaunchpadWriteTarFile.files_to_stream(files) |
277 | 413 | self.import_queue.addOrUpdateEntriesFromTarball( | 413 | self.import_queue.addOrUpdateEntriesFromTarball( |
278 | 414 | tarfile_content, True, self.importer, | 414 | tarfile_content, True, self.importer, |
279 | 415 | productseries=self.productseries) | 415 | productseries=self.productseries) |
280 | @@ -420,7 +420,7 @@ | |||
281 | 420 | files = dict(( | 420 | files = dict(( |
282 | 421 | self._makeFile(), | 421 | self._makeFile(), |
283 | 422 | )) | 422 | )) |
285 | 423 | tarfile_content = LaunchpadWriteTarFile.files_to_string(files) | 423 | tarfile_content = LaunchpadWriteTarFile.files_to_stream(files) |
286 | 424 | self.import_queue.addOrUpdateEntriesFromTarball( | 424 | self.import_queue.addOrUpdateEntriesFromTarball( |
287 | 425 | tarfile_content, True, self.importer, | 425 | tarfile_content, True, self.importer, |
288 | 426 | productseries=self.productseries) | 426 | productseries=self.productseries) |
289 | @@ -431,7 +431,7 @@ | |||
290 | 431 | files = dict(( | 431 | files = dict(( |
291 | 432 | self._makeFile('pot', 'directory'), | 432 | self._makeFile('pot', 'directory'), |
292 | 433 | )) | 433 | )) |
294 | 434 | tarfile_content = LaunchpadWriteTarFile.files_to_string(files) | 434 | tarfile_content = LaunchpadWriteTarFile.files_to_stream(files) |
295 | 435 | self.import_queue.addOrUpdateEntriesFromTarball( | 435 | self.import_queue.addOrUpdateEntriesFromTarball( |
296 | 436 | tarfile_content, True, self.importer, | 436 | tarfile_content, True, self.importer, |
297 | 437 | productseries=self.productseries) | 437 | productseries=self.productseries) |
298 | @@ -441,7 +441,7 @@ | |||
299 | 441 | # Leading slashes are stripped from path names. | 441 | # Leading slashes are stripped from path names. |
300 | 442 | path, content = self._makeFile('pot', '/directory') | 442 | path, content = self._makeFile('pot', '/directory') |
301 | 443 | files = dict(((path, content),)) | 443 | files = dict(((path, content),)) |
303 | 444 | tarfile_content = LaunchpadWriteTarFile.files_to_string(files) | 444 | tarfile_content = LaunchpadWriteTarFile.files_to_stream(files) |
304 | 445 | self.import_queue.addOrUpdateEntriesFromTarball( | 445 | self.import_queue.addOrUpdateEntriesFromTarball( |
305 | 446 | tarfile_content, True, self.importer, | 446 | tarfile_content, True, self.importer, |
306 | 447 | productseries=self.productseries) | 447 | productseries=self.productseries) |
307 | 448 | 448 | ||
308 | === modified file 'lib/lp/translations/utilities/kde_po_importer.py' | |||
309 | --- lib/lp/translations/utilities/kde_po_importer.py 2015-07-08 16:05:11 +0000 | |||
310 | +++ lib/lp/translations/utilities/kde_po_importer.py 2017-04-24 14:03:06 +0000 | |||
311 | @@ -1,4 +1,4 @@ | |||
313 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
314 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
315 | 3 | 3 | ||
316 | 4 | """Import module for legacy KDE .po files. | 4 | """Import module for legacy KDE .po files. |
317 | @@ -41,11 +41,17 @@ | |||
318 | 41 | # and with extremely big PO files, this will be too slow). Thus, | 41 | # and with extremely big PO files, this will be too slow). Thus, |
319 | 42 | # a heuristic verified to be correct on all PO files from | 42 | # a heuristic verified to be correct on all PO files from |
320 | 43 | # Ubuntu language packs. | 43 | # Ubuntu language packs. |
326 | 44 | if ('msgid "_n: ' in file_contents or | 44 | msgid_start = False |
327 | 45 | 'msgid ""\n"_n: ' in file_contents or | 45 | for line in file_contents: |
328 | 46 | 'msgid "_: ' in file_contents or | 46 | if line == b'msgid ""\n': |
329 | 47 | 'msgid ""\n"_: ' in file_contents): | 47 | msgid_start = True |
330 | 48 | return TranslationFileFormat.KDEPO | 48 | elif (line.startswith(b'msgid "_n: ') or |
331 | 49 | (msgid_start and line.startswith(b'"_n: ')) or | ||
332 | 50 | line.startswith(b'msgid "_: ') or | ||
333 | 51 | (msgid_start and line.startswith(b'"_: '))): | ||
334 | 52 | return TranslationFileFormat.KDEPO | ||
335 | 53 | else: | ||
336 | 54 | msgid_start = False | ||
337 | 49 | else: | 55 | else: |
338 | 50 | return TranslationFileFormat.PO | 56 | return TranslationFileFormat.PO |
339 | 51 | 57 | ||
340 | 52 | 58 | ||
341 | === modified file 'lib/lp/translations/utilities/tests/helpers.py' | |||
342 | --- lib/lp/translations/utilities/tests/helpers.py 2011-09-18 08:39:01 +0000 | |||
343 | +++ lib/lp/translations/utilities/tests/helpers.py 2017-04-24 14:03:06 +0000 | |||
344 | @@ -1,4 +1,4 @@ | |||
346 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
347 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
348 | 3 | 3 | ||
349 | 4 | """Helper module reused in different tests.""" | 4 | """Helper module reused in different tests.""" |
350 | @@ -25,7 +25,8 @@ | |||
351 | 25 | pofile=None, potemplate=None, by_maintainer=True): | 25 | pofile=None, potemplate=None, by_maintainer=True): |
352 | 26 | """Import a `POFile` or `POTemplate` from the given string. | 26 | """Import a `POFile` or `POTemplate` from the given string. |
353 | 27 | 27 | ||
355 | 28 | :param file_contents: text of "file" to import. | 28 | :param file_contents: text of "file" to import, or a seekable file |
356 | 29 | object containing the text. | ||
357 | 29 | :param person: party requesting the import. | 30 | :param person: party requesting the import. |
358 | 30 | :param pofile: if uploading a `POFile`, file to import to; None otherwise. | 31 | :param pofile: if uploading a `POFile`, file to import to; None otherwise. |
359 | 31 | :param potemplate: if uploading a `POTemplate`, file to import to; None | 32 | :param potemplate: if uploading a `POTemplate`, file to import to; None |
360 | 32 | 33 | ||
361 | === modified file 'lib/lp/translations/utilities/tests/test_gettext_po_importer.py' | |||
362 | --- lib/lp/translations/utilities/tests/test_gettext_po_importer.py 2012-01-01 02:58:52 +0000 | |||
363 | +++ lib/lp/translations/utilities/tests/test_gettext_po_importer.py 2017-04-24 14:03:06 +0000 | |||
364 | @@ -1,10 +1,11 @@ | |||
366 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
367 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
368 | 3 | 3 | ||
369 | 4 | """Gettext PO importer tests.""" | 4 | """Gettext PO importer tests.""" |
370 | 5 | 5 | ||
371 | 6 | __metaclass__ = type | 6 | __metaclass__ = type |
372 | 7 | 7 | ||
373 | 8 | from io import BytesIO | ||
374 | 8 | import unittest | 9 | import unittest |
375 | 9 | 10 | ||
376 | 10 | import transaction | 11 | import transaction |
377 | @@ -89,7 +90,7 @@ | |||
378 | 89 | 90 | ||
379 | 90 | def testFormat(self): | 91 | def testFormat(self): |
380 | 91 | # GettextPOImporter reports that it handles the PO file format. | 92 | # GettextPOImporter reports that it handles the PO file format. |
382 | 92 | format = self.template_importer.getFormat(test_template) | 93 | format = self.template_importer.getFormat(BytesIO(test_template)) |
383 | 93 | self.failUnless( | 94 | self.failUnless( |
384 | 94 | format == TranslationFileFormat.PO, | 95 | format == TranslationFileFormat.PO, |
385 | 95 | 'GettextPOImporter format expected PO but got %s' % format.name) | 96 | 'GettextPOImporter format expected PO but got %s' % format.name) |
386 | 96 | 97 | ||
387 | === modified file 'lib/lp/translations/utilities/tests/test_kde_po_importer.py' | |||
388 | --- lib/lp/translations/utilities/tests/test_kde_po_importer.py 2012-01-01 02:58:52 +0000 | |||
389 | +++ lib/lp/translations/utilities/tests/test_kde_po_importer.py 2017-04-24 14:03:06 +0000 | |||
390 | @@ -1,10 +1,11 @@ | |||
392 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
393 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
394 | 3 | 3 | ||
395 | 4 | """KDE PO importer tests.""" | 4 | """KDE PO importer tests.""" |
396 | 5 | 5 | ||
397 | 6 | __metaclass__ = type | 6 | __metaclass__ = type |
398 | 7 | 7 | ||
399 | 8 | from io import BytesIO | ||
400 | 8 | import unittest | 9 | import unittest |
401 | 9 | 10 | ||
402 | 10 | import transaction | 11 | import transaction |
403 | @@ -114,7 +115,7 @@ | |||
404 | 114 | 115 | ||
405 | 115 | def testFormat(self): | 116 | def testFormat(self): |
406 | 116 | """Check whether KdePOImporter can handle the KDEPO file format.""" | 117 | """Check whether KdePOImporter can handle the KDEPO file format.""" |
408 | 117 | format = self.template_importer.getFormat(test_kde_template) | 118 | format = self.template_importer.getFormat(BytesIO(test_kde_template)) |
409 | 118 | self.failUnless( | 119 | self.failUnless( |
410 | 119 | format == TranslationFileFormat.KDEPO, | 120 | format == TranslationFileFormat.KDEPO, |
411 | 120 | 'KdePOImporter format expected KDEPO but got %s' % format.name) | 121 | 'KdePOImporter format expected KDEPO but got %s' % format.name) |
412 | 121 | 122 | ||
413 | === modified file 'lib/lp/translations/utilities/tests/test_mozilla_xpi_importer.py' | |||
414 | --- lib/lp/translations/utilities/tests/test_mozilla_xpi_importer.py 2011-12-28 17:03:06 +0000 | |||
415 | +++ lib/lp/translations/utilities/tests/test_mozilla_xpi_importer.py 2017-04-24 14:03:06 +0000 | |||
416 | @@ -1,10 +1,11 @@ | |||
418 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
419 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
420 | 3 | 3 | ||
421 | 4 | """Mozilla XPI importer tests.""" | 4 | """Mozilla XPI importer tests.""" |
422 | 5 | 5 | ||
423 | 6 | __metaclass__ = type | 6 | __metaclass__ = type |
424 | 7 | 7 | ||
425 | 8 | from io import BytesIO | ||
426 | 8 | import unittest | 9 | import unittest |
427 | 9 | 10 | ||
428 | 10 | from zope.interface.verify import verifyObject | 11 | from zope.interface.verify import verifyObject |
429 | @@ -34,7 +35,7 @@ | |||
430 | 34 | 35 | ||
431 | 35 | def testFormat(self): | 36 | def testFormat(self): |
432 | 36 | """Check that MozillaXpiImporter handles the XPI file format.""" | 37 | """Check that MozillaXpiImporter handles the XPI file format.""" |
434 | 37 | format = self.importer.getFormat(u'') | 38 | format = self.importer.getFormat(BytesIO(b'')) |
435 | 38 | self.failUnless( | 39 | self.failUnless( |
436 | 39 | format == TranslationFileFormat.XPI, | 40 | format == TranslationFileFormat.XPI, |
437 | 40 | 'MozillaXpiImporter format expected XPI but got %s' % format.name) | 41 | 'MozillaXpiImporter format expected XPI but got %s' % format.name) |
438 | 41 | 42 | ||
439 | === modified file 'lib/lp/translations/utilities/tests/test_translation_importer.py' | |||
440 | --- lib/lp/translations/utilities/tests/test_translation_importer.py 2012-01-01 02:58:52 +0000 | |||
441 | +++ lib/lp/translations/utilities/tests/test_translation_importer.py 2017-04-24 14:03:06 +0000 | |||
442 | @@ -1,10 +1,12 @@ | |||
444 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
445 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
446 | 3 | 3 | ||
447 | 4 | """Translation Importer tests.""" | 4 | """Translation Importer tests.""" |
448 | 5 | 5 | ||
449 | 6 | __metaclass__ = type | 6 | __metaclass__ = type |
450 | 7 | 7 | ||
451 | 8 | from io import BytesIO | ||
452 | 9 | |||
453 | 8 | import transaction | 10 | import transaction |
454 | 9 | 11 | ||
455 | 10 | from lp.services.log.logger import DevNullLogger | 12 | from lp.services.log.logger import DevNullLogger |
456 | @@ -84,17 +86,17 @@ | |||
457 | 84 | self.assertEqual( | 86 | self.assertEqual( |
458 | 85 | TranslationFileFormat.PO, | 87 | TranslationFileFormat.PO, |
459 | 86 | importer.getTranslationFileFormat( | 88 | importer.getTranslationFileFormat( |
461 | 87 | ".po", u'msgid "message"\nmsgstr ""')) | 89 | ".po", BytesIO(b'msgid "message"\nmsgstr ""'))) |
462 | 88 | 90 | ||
463 | 89 | # And PO file with KDE-style messages is recognised as KDEPO file. | 91 | # And PO file with KDE-style messages is recognised as KDEPO file. |
464 | 90 | self.assertEqual( | 92 | self.assertEqual( |
465 | 91 | TranslationFileFormat.KDEPO, | 93 | TranslationFileFormat.KDEPO, |
466 | 92 | importer.getTranslationFileFormat( | 94 | importer.getTranslationFileFormat( |
468 | 93 | ".po", u'msgid "_: kde context\nmessage"\nmsgstr ""')) | 95 | ".po", BytesIO(b'msgid "_: kde context\nmessage"\nmsgstr ""'))) |
469 | 94 | 96 | ||
470 | 95 | self.assertEqual( | 97 | self.assertEqual( |
471 | 96 | TranslationFileFormat.XPI, | 98 | TranslationFileFormat.XPI, |
473 | 97 | importer.getTranslationFileFormat(".xpi", u"")) | 99 | importer.getTranslationFileFormat(".xpi", BytesIO(b""))) |
474 | 98 | 100 | ||
475 | 99 | def testNoConflictingPriorities(self): | 101 | def testNoConflictingPriorities(self): |
476 | 100 | """Check that no two importers for the same file extension have | 102 | """Check that no two importers for the same file extension have |
477 | 101 | 103 | ||
478 | === modified file 'lib/lp/translations/utilities/tests/test_xpi_import.py' | |||
479 | --- lib/lp/translations/utilities/tests/test_xpi_import.py 2012-01-01 02:58:52 +0000 | |||
480 | +++ lib/lp/translations/utilities/tests/test_xpi_import.py 2017-04-24 14:03:06 +0000 | |||
481 | @@ -1,4 +1,4 @@ | |||
483 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
484 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
485 | 3 | 3 | ||
486 | 4 | """Functional tests for XPI file format""" | 4 | """Functional tests for XPI file format""" |
487 | @@ -61,7 +61,7 @@ | |||
488 | 61 | # Get the file to import. | 61 | # Get the file to import. |
489 | 62 | en_US_xpi = get_en_US_xpi_file_to_import(subdir) | 62 | en_US_xpi = get_en_US_xpi_file_to_import(subdir) |
490 | 63 | return import_pofile_or_potemplate( | 63 | return import_pofile_or_potemplate( |
492 | 64 | file_contents=en_US_xpi.read(), | 64 | file_contents=en_US_xpi, |
493 | 65 | person=self.importer, | 65 | person=self.importer, |
494 | 66 | potemplate=self.firefox_template) | 66 | potemplate=self.firefox_template) |
495 | 67 | 67 | ||
496 | @@ -74,7 +74,7 @@ | |||
497 | 74 | # just use the same template file like a translation one. | 74 | # just use the same template file like a translation one. |
498 | 75 | es_xpi = get_en_US_xpi_file_to_import(subdir) | 75 | es_xpi = get_en_US_xpi_file_to_import(subdir) |
499 | 76 | return import_pofile_or_potemplate( | 76 | return import_pofile_or_potemplate( |
501 | 77 | file_contents=es_xpi.read(), | 77 | file_contents=es_xpi, |
502 | 78 | person=self.importer, | 78 | person=self.importer, |
503 | 79 | pofile=self.spanish_firefox, | 79 | pofile=self.spanish_firefox, |
504 | 80 | by_maintainer=True) | 80 | by_maintainer=True) |
505 | 81 | 81 | ||
506 | === modified file 'lib/lp/translations/utilities/tests/test_xpi_po_exporter.py' | |||
507 | --- lib/lp/translations/utilities/tests/test_xpi_po_exporter.py 2012-03-27 13:41:38 +0000 | |||
508 | +++ lib/lp/translations/utilities/tests/test_xpi_po_exporter.py 2017-04-24 14:03:06 +0000 | |||
509 | @@ -1,4 +1,4 @@ | |||
511 | 1 | # Copyright 2009-2012 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
512 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
513 | 3 | 3 | ||
514 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
515 | @@ -82,7 +82,7 @@ | |||
516 | 82 | translation_import_queue = getUtility(ITranslationImportQueue) | 82 | translation_import_queue = getUtility(ITranslationImportQueue) |
517 | 83 | by_maintainer = True | 83 | by_maintainer = True |
518 | 84 | entry = translation_import_queue.addOrUpdateEntry( | 84 | entry = translation_import_queue.addOrUpdateEntry( |
520 | 85 | self.firefox_template.path, en_US_xpi.read(), by_maintainer, | 85 | self.firefox_template.path, en_US_xpi, by_maintainer, |
521 | 86 | self.importer, productseries=self.firefox_template.productseries, | 86 | self.importer, productseries=self.firefox_template.productseries, |
522 | 87 | potemplate=self.firefox_template) | 87 | potemplate=self.firefox_template) |
523 | 88 | 88 | ||
524 | 89 | 89 | ||
525 | === modified file 'lib/lp/translations/utilities/tests/test_xpi_search.py' | |||
526 | --- lib/lp/translations/utilities/tests/test_xpi_search.py 2012-01-01 02:58:52 +0000 | |||
527 | +++ lib/lp/translations/utilities/tests/test_xpi_search.py 2017-04-24 14:03:06 +0000 | |||
528 | @@ -1,4 +1,4 @@ | |||
530 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
531 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
532 | 3 | 3 | ||
533 | 4 | """Functional tests for searching through XPI POTemplates""" | 4 | """Functional tests for searching through XPI POTemplates""" |
534 | @@ -51,7 +51,7 @@ | |||
535 | 51 | # Get the file to import. | 51 | # Get the file to import. |
536 | 52 | en_US_xpi = get_en_US_xpi_file_to_import(subdir) | 52 | en_US_xpi = get_en_US_xpi_file_to_import(subdir) |
537 | 53 | return import_pofile_or_potemplate( | 53 | return import_pofile_or_potemplate( |
539 | 54 | file_contents=en_US_xpi.read(), | 54 | file_contents=en_US_xpi, |
540 | 55 | person=self.importer, | 55 | person=self.importer, |
541 | 56 | potemplate=self.firefox_template) | 56 | potemplate=self.firefox_template) |
542 | 57 | 57 |