Merge lp:~lifeless/python-oops-datedir-repo/extraction into lp:python-oops-datedir-repo
- extraction
- Merge into trunk
Proposed by
Robert Collins
Status: | Merged |
---|---|
Merged at revision: | 7 |
Proposed branch: | lp:~lifeless/python-oops-datedir-repo/extraction |
Merge into: | lp:python-oops-datedir-repo |
Diff against target: |
327 lines (+114/-45) 4 files modified
oops_datedir_repo/__init__.py (+1/-1) oops_datedir_repo/serializer_rfc822.py (+43/-33) oops_datedir_repo/tests/test_serializer_rfc822.py (+69/-10) setup.py (+1/-1) |
To merge this branch: | bzr merge lp:~lifeless/python-oops-datedir-repo/extraction |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Steve Kowalik (community) | code | Approve | |
Review via email: mp+71795@code.launchpad.net |
Commit message
Description of the change
This handles the new topic and reporter keys (mapping pageid to topic on in-out for compat with oops-tools (until oops-tools uses this library :P)), and takes more encoding responsibility so that callers can relax a little (given the definition we're aiming at of 'bson serializable is all we need).
To post a comment you must log in.
Revision history for this message
Steve Kowalik (stevenk) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'oops_datedir_repo/__init__.py' |
2 | --- oops_datedir_repo/__init__.py 2011-08-16 02:21:20 +0000 |
3 | +++ oops_datedir_repo/__init__.py 2011-08-17 01:17:24 +0000 |
4 | @@ -25,7 +25,7 @@ |
5 | # established at this point, and setup.py will use a version of next-$(revno). |
6 | # If the releaselevel is 'final', then the tarball will be major.minor.micro. |
7 | # Otherwise it is major.minor.micro~$(revno). |
8 | -__version__ = (0, 0, 2, 'beta', 0) |
9 | +__version__ = (0, 0, 3, 'beta', 0) |
10 | |
11 | __all__ = [ |
12 | 'DateDirRepo', |
13 | |
14 | === modified file 'oops_datedir_repo/serializer_rfc822.py' |
15 | --- oops_datedir_repo/serializer_rfc822.py 2011-08-15 05:30:39 +0000 |
16 | +++ oops_datedir_repo/serializer_rfc822.py 2011-08-17 01:17:24 +0000 |
17 | @@ -19,13 +19,16 @@ |
18 | This style of OOPS format is very web server specific, not extensible - it |
19 | should be considered deprecated. |
20 | |
21 | -The reports this serializer handles always have the following variables: |
22 | +The reports this serializer handles always have the following variables (See |
23 | +the python-oops api docs for more information about these variables): |
24 | |
25 | * id: The name of this error report. |
26 | * type: The type of the exception that occurred. |
27 | * value: The value of the exception that occurred. |
28 | * time: The time at which the exception occurred. |
29 | -* pageid: The identifier for the template/script that oopsed. |
30 | +* reporter: The reporting program. |
31 | +* topic: The identifier for the template/script that oopsed. |
32 | + [this is written as Page-Id for compatibility with as yet unported tools.] |
33 | * branch_nick: The branch nickname. |
34 | * revno: The revision number of the branch. |
35 | * tb_text: A text version of the traceback. |
36 | @@ -37,6 +40,9 @@ |
37 | * revno: The revision that the branch was at. |
38 | * Informational: A flag, True if the error wasn't fatal- if it was |
39 | 'informational'. |
40 | + [Deprecated - this is no longer part of the oops report conventions. Existing |
41 | + reports with it set are still read, but the key is only present if it was |
42 | + truely in the report.] |
43 | """ |
44 | |
45 | |
46 | @@ -67,13 +73,16 @@ |
47 | date =iso8601.parse_date(msg.getheader('date')) |
48 | else: |
49 | date = None |
50 | - pageid = msg.getheader('page-id') |
51 | + topic = msg.getheader('topic') |
52 | + if topic is None: |
53 | + topic = msg.getheader('page-id') |
54 | username = msg.getheader('user') |
55 | url = msg.getheader('url') |
56 | duration = int(float(msg.getheader('duration', '-1'))) |
57 | informational = msg.getheader('informational') |
58 | branch_nick = msg.getheader('branch') |
59 | revno = msg.getheader('revision') |
60 | + reporter = msg.getheader('oops-reporter') |
61 | |
62 | # Explicitly use an iterator so we can process the file |
63 | # sequentially. In most instances the iterator will actually |
64 | @@ -109,10 +118,15 @@ |
65 | # The rest is traceback. |
66 | tb_text = ''.join(lines) |
67 | |
68 | - return dict(id=id, type=exc_type, value=exc_value, time=date, |
69 | - pageid=pageid, tb_text=tb_text, username=username, url=url, |
70 | + result = dict(id=id, type=exc_type, value=exc_value, time=date, |
71 | + topic=topic, tb_text=tb_text, username=username, url=url, |
72 | duration=duration, req_vars=req_vars, db_statements=statements, |
73 | - informational=informational, branch_nick=branch_nick, revno=revno) |
74 | + branch_nick=branch_nick, revno=revno) |
75 | + if informational is not None: |
76 | + result['informational'] = informational |
77 | + if reporter is not None: |
78 | + result['reporter'] = reporter |
79 | + return result |
80 | |
81 | |
82 | def _normalise_whitespace(s): |
83 | @@ -151,45 +165,41 @@ |
84 | def to_chunks(report): |
85 | """Returns a list of bytestrings making up the serialized oops.""" |
86 | chunks = [] |
87 | - chunks.append('Oops-Id: %s\n' % _normalise_whitespace(report['id'])) |
88 | - if 'type' in report: |
89 | - chunks.append( |
90 | - 'Exception-Type: %s\n' % _normalise_whitespace(report['type'])) |
91 | - if 'value' in report: |
92 | - chunks.append( |
93 | - 'Exception-Value: %s\n' % _normalise_whitespace(report['value'])) |
94 | + def header(label, key, optional=True): |
95 | + if optional and key not in report: |
96 | + return |
97 | + value = _safestr(report[key]) |
98 | + value = _normalise_whitespace(value) |
99 | + chunks.append('%s: %s\n' % (label, value)) |
100 | + header('Oops-Id', 'id', optional=False) |
101 | + header('Exception-Type', 'type') |
102 | + header('Exception-Value', 'value') |
103 | if 'time' in report: |
104 | chunks.append('Date: %s\n' % report['time'].isoformat()) |
105 | - if 'pageid' in report: |
106 | - chunks.append( |
107 | - 'Page-Id: %s\n' % _normalise_whitespace(report['pageid'])) |
108 | - if 'branch_nick' in report: |
109 | - chunks.append('Branch: %s\n' % _safestr(report['branch_nick'])) |
110 | - if 'revno' in report: |
111 | - chunks.append('Revision: %s\n' % report['revno']) |
112 | - if 'username' in report: |
113 | - chunks.append( |
114 | - 'User: %s\n' % _normalise_whitespace(report['username'])) |
115 | - if 'url' in report: |
116 | - chunks.append('URL: %s\n' % _normalise_whitespace(report['url'])) |
117 | - if 'duration' in report: |
118 | - chunks.append('Duration: %s\n' % report['duration']) |
119 | - if 'informational' in report: |
120 | - chunks.append('Informational: %s\n' % report['informational']) |
121 | + header('Page-Id', 'topic') |
122 | + header('Branch', 'branch_nick') |
123 | + header('Revision', 'revno') |
124 | + header('User', 'username') |
125 | + header('URL', 'url') |
126 | + header('Duration', 'duration') |
127 | + header('Informational', 'informational') |
128 | + header('Oops-Reporter', 'reporter') |
129 | chunks.append('\n') |
130 | safe_chars = ';/\\?:@&+$, ()*!' |
131 | if 'req_vars' in report: |
132 | for key, value in report['req_vars']: |
133 | - chunks.append('%s=%s\n' % (urllib.quote(key, safe_chars), |
134 | - urllib.quote(value, safe_chars))) |
135 | + chunks.append('%s=%s\n' % ( |
136 | + urllib.quote(_safestr(key), safe_chars), |
137 | + urllib.quote(_safestr(value), safe_chars))) |
138 | chunks.append('\n') |
139 | if 'db_statements' in report: |
140 | for (start, end, database_id, statement) in report['db_statements']: |
141 | chunks.append('%05d-%05d@%s %s\n' % ( |
142 | - start, end, database_id, _normalise_whitespace(statement))) |
143 | + start, end, _safestr(database_id), |
144 | + _safestr(_normalise_whitespace(statement)))) |
145 | chunks.append('\n') |
146 | if 'tb_text' in report: |
147 | - chunks.append(report['tb_text']) |
148 | + chunks.append(_safestr(report['tb_text'])) |
149 | return chunks |
150 | |
151 | |
152 | |
153 | === modified file 'oops_datedir_repo/tests/test_serializer_rfc822.py' |
154 | --- oops_datedir_repo/tests/test_serializer_rfc822.py 2011-08-15 05:30:39 +0000 |
155 | +++ oops_datedir_repo/tests/test_serializer_rfc822.py 2011-08-17 01:17:24 +0000 |
156 | @@ -41,7 +41,7 @@ |
157 | Exception-Type: NotFound |
158 | Exception-Value: error message |
159 | Date: 2005-04-01T00:00:00+00:00 |
160 | - Page-Id: IFoo:+foo-template |
161 | + Topic: IFoo:+foo-template |
162 | User: Sample User |
163 | URL: http://localhost:9000/foo |
164 | Duration: 42 |
165 | @@ -60,7 +60,7 @@ |
166 | self.assertEqual(report['value'], 'error message') |
167 | self.assertEqual( |
168 | report['time'], datetime.datetime(2005, 4, 1, tzinfo=utc)) |
169 | - self.assertEqual(report['pageid'], 'IFoo:+foo-template') |
170 | + self.assertEqual(report['topic'], 'IFoo:+foo-template') |
171 | self.assertEqual(report['tb_text'], 'traceback-text') |
172 | self.assertEqual(report['username'], 'Sample User') |
173 | self.assertEqual(report['url'], 'http://localhost:9000/foo') |
174 | @@ -86,7 +86,7 @@ |
175 | Exception-Type: NotFound |
176 | Exception-Value: error message |
177 | Date: 2005-04-01T00:00:00+00:00 |
178 | - Page-Id: IFoo:+foo-template |
179 | + Topic: IFoo:+foo-template |
180 | User: Sample User |
181 | URL: http://localhost:9000/foo |
182 | Duration: 42 |
183 | @@ -105,7 +105,7 @@ |
184 | self.assertEqual(report['value'], 'error message') |
185 | self.assertEqual( |
186 | report['time'], datetime.datetime(2005, 4, 1, tzinfo=utc)) |
187 | - self.assertEqual(report['pageid'], 'IFoo:+foo-template') |
188 | + self.assertEqual(report['topic'], 'IFoo:+foo-template') |
189 | self.assertEqual(report['tb_text'], 'traceback-text') |
190 | self.assertEqual(report['username'], 'Sample User') |
191 | self.assertEqual(report['url'], 'http://localhost:9000/foo') |
192 | @@ -119,7 +119,6 @@ |
193 | self.assertEqual(len(report['db_statements']), 2) |
194 | self.assertEqual(report['db_statements'][0], (1, 5, None, 'SELECT 1')) |
195 | self.assertEqual(report['db_statements'][1], (5, 10, None, 'SELECT 2')) |
196 | - self.assertFalse(report['informational']) |
197 | |
198 | def test_read_branch_nick_revno(self): |
199 | """Test ErrorReport.read().""" |
200 | @@ -128,7 +127,6 @@ |
201 | Exception-Type: NotFound |
202 | Exception-Value: error message |
203 | Date: 2005-04-01T00:00:00+00:00 |
204 | - Page-Id: IFoo:+foo-template |
205 | User: Sample User |
206 | URL: http://localhost:9000/foo |
207 | Duration: 42 |
208 | @@ -147,6 +145,45 @@ |
209 | self.assertEqual(report['branch_nick'], 'mybranch') |
210 | self.assertEqual(report['revno'], '45') |
211 | |
212 | + def test_read_reporter(self): |
213 | + """Test ErrorReport.read().""" |
214 | + fp = StringIO.StringIO(dedent("""\ |
215 | + Oops-Id: OOPS-A0001 |
216 | + Oops-Reporter: foo/bar |
217 | + |
218 | + """)) |
219 | + report = read(fp) |
220 | + self.assertEqual(report['reporter'], 'foo/bar') |
221 | + |
222 | + def test_read_pageid_to_topic(self): |
223 | + """Test ErrorReport.read().""" |
224 | + fp = StringIO.StringIO(dedent("""\ |
225 | + Oops-Id: OOPS-A0001 |
226 | + Page-Id: IFoo:+foo-template |
227 | + |
228 | + """)) |
229 | + report = read(fp) |
230 | + self.assertEqual(report['topic'], 'IFoo:+foo-template') |
231 | + |
232 | + def test_read_informational_read(self): |
233 | + """Test ErrorReport.read().""" |
234 | + fp = StringIO.StringIO(dedent("""\ |
235 | + Oops-Id: OOPS-A0001 |
236 | + Informational: True |
237 | + |
238 | + """)) |
239 | + report = read(fp) |
240 | + self.assertEqual('True', report['informational']) |
241 | + |
242 | + def test_read_no_informational_no_key(self): |
243 | + """Test ErrorReport.read().""" |
244 | + fp = StringIO.StringIO(dedent("""\ |
245 | + Oops-Id: OOPS-A0001 |
246 | + |
247 | + """)) |
248 | + report = read(fp) |
249 | + self.assertFalse('informational' in report) |
250 | + |
251 | def test_minimal_oops(self): |
252 | # If we get a crazy-small oops, we can read it sensibly. Because there |
253 | # is existing legacy code, all keys are filled in with None, [] rather |
254 | @@ -159,14 +196,13 @@ |
255 | self.assertEqual(report['type'], None) |
256 | self.assertEqual(report['value'], None) |
257 | self.assertEqual(report['time'], None) |
258 | - self.assertEqual(report['pageid'], None) |
259 | + self.assertEqual(report['topic'], None) |
260 | self.assertEqual(report['tb_text'], '') |
261 | self.assertEqual(report['username'], None) |
262 | self.assertEqual(report['url'], None) |
263 | self.assertEqual(report['duration'], -1) |
264 | self.assertEqual(len(report['req_vars']), 0) |
265 | self.assertEqual(len(report['db_statements']), 0) |
266 | - self.assertFalse(report['informational']) |
267 | self.assertEqual(report['branch_nick'], None) |
268 | self.assertEqual(report['revno'], None) |
269 | |
270 | @@ -180,7 +216,7 @@ |
271 | 'type': 'NotFound', |
272 | 'value': 'error message', |
273 | 'time': datetime.datetime(2005, 04, 01, 00, 00, 00, tzinfo=utc), |
274 | - 'pageid': 'IFoo:+foo-template', |
275 | + 'topic': 'IFoo:+foo-template', |
276 | 'tb_text': 'traceback-text', |
277 | 'username': 'Sample User', |
278 | 'url': 'http://localhost:9000/foo', |
279 | @@ -223,7 +259,7 @@ |
280 | 'type': 'NotFound', |
281 | 'value': 'error message', |
282 | 'time': datetime.datetime(2005, 04, 01, 00, 00, 00, tzinfo=utc), |
283 | - 'pageid': 'IFoo:+foo-template', |
284 | + 'topic': 'IFoo:+foo-template', |
285 | 'tb_text': 'traceback-text', |
286 | 'username': 'Sample User', |
287 | 'url': 'http://localhost:9000/foo', |
288 | @@ -269,3 +305,26 @@ |
289 | "Oops-Id: OOPS-1234\n", |
290 | "\n" |
291 | ], to_chunks(report)) |
292 | + |
293 | + def test_reporter(self): |
294 | + report = {'reporter': 'foo', 'id': 'bar'} |
295 | + self.assertEqual([ |
296 | + "Oops-Id: bar\n", |
297 | + "Oops-Reporter: foo\n", |
298 | + "\n", |
299 | + ], to_chunks(report)) |
300 | + |
301 | + def test_bad_strings(self): |
302 | + # Because of the rfc822 limitations, not all strings can be supported |
303 | + # in this format - particularly in headers... so all header strings are |
304 | + # passed through an escape process. |
305 | + report = {'id': u'\xeafoo'} |
306 | + self.assertEqual([ |
307 | + "Oops-Id: \\xeafoo\n", |
308 | + "\n", |
309 | + ], to_chunks(report)) |
310 | + report = {'id': '\xeafoo'} |
311 | + self.assertEqual([ |
312 | + "Oops-Id: \\xeafoo\n", |
313 | + "\n", |
314 | + ], to_chunks(report)) |
315 | |
316 | === modified file 'setup.py' |
317 | --- setup.py 2011-08-16 02:21:20 +0000 |
318 | +++ setup.py 2011-08-17 01:17:24 +0000 |
319 | @@ -22,7 +22,7 @@ |
320 | description = file(os.path.join(os.path.dirname(__file__), 'README'), 'rb').read() |
321 | |
322 | setup(name="oops_datedir_repo", |
323 | - version="0.0.2", |
324 | + version="0.0.3", |
325 | description="OOPS disk serialisation and repository management.", |
326 | long_description=description, |
327 | maintainer="Launchpad Developers", |