Merge lp:~gz/brz/py3_bencode_pyx into lp:brz

Proposed by Martin Packman on 2017-06-27
Status: Merged
Approved by: Martin Packman on 2017-06-27
Approved revision: 6725
Merge reported by: The Breezy Bot
Merged at revision: not available
Proposed branch: lp:~gz/brz/py3_bencode_pyx
Merge into: lp:brz
Prerequisite: lp:~gz/brz/py3_static_tuple_import
Diff against target: 237 lines (+64/-54)
1 file modified
breezy/_bencode_pyx.pyx (+64/-54)
To merge this branch: bzr merge lp:~gz/brz/py3_bencode_pyx
Reviewer Review Type Date Requested Status
Jelmer Vernooij 2017-06-27 Approve on 2017-06-27
Review via email: mp+326331@code.launchpad.net

Commit message

Make _bencode_pyx compile and pass tests on Python 3

Description of the change

A bunch of changes to modern cython style, which gets us free aliasing of the PyBytes names and such like.

One of the few tricky bits is the Int/Long switching, which should be sane. On encode, it's just a fast path, so now uses bit_length() to check the size rather than just inferring from type. (Trivia, bit_length doesn't do 2's compliment.) On decode PyInt_FromString will promote to long on Python 2, and is always long on 3.

To post a comment you must log in.
Jelmer Vernooij (jelmer) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'breezy/_bencode_pyx.pyx'
--- breezy/_bencode_pyx.pyx 2017-06-10 01:43:31 +0000
+++ breezy/_bencode_pyx.pyx 2017-06-27 01:35:34 +0000
@@ -18,38 +18,51 @@
1818
19from __future__ import absolute_import19from __future__ import absolute_import
2020
21from cpython.bytes cimport (
22 PyBytes_CheckExact,
23 PyBytes_FromStringAndSize,
24 PyBytes_AS_STRING,
25 PyBytes_GET_SIZE,
26 )
27from cpython.long cimport (
28 PyLong_CheckExact,
29 )
30from cpython.int cimport (
31 PyInt_CheckExact,
32 PyInt_FromString,
33 )
34from cpython.tuple cimport (
35 PyTuple_CheckExact,
36 )
37from cpython.list cimport (
38 PyList_CheckExact,
39 PyList_Append,
40 )
41from cpython.dict cimport (
42 PyDict_CheckExact,
43 )
44from cpython.bool cimport (
45 PyBool_Check,
46 )
47from cpython.mem cimport (
48 PyMem_Free,
49 PyMem_Malloc,
50 PyMem_Realloc,
51 )
2152
22cdef extern from "stddef.h":53from libc.stdlib cimport (
23 ctypedef unsigned int size_t54 strtol,
55 )
56from libc.string cimport (
57 memcpy,
58 )
2459
25cdef extern from "Python.h":60cdef extern from "Python.h":
26 ctypedef int Py_ssize_t61 # There is no cython module for ceval.h for some reason
27 int PyInt_CheckExact(object o)
28 int PyLong_CheckExact(object o)
29 int PyString_CheckExact(object o)
30 int PyTuple_CheckExact(object o)
31 int PyList_CheckExact(object o)
32 int PyDict_CheckExact(object o)
33 int PyBool_Check(object o)
34 object PyString_FromStringAndSize(char *v, Py_ssize_t len)
35 char *PyString_AS_STRING(object o) except NULL
36 Py_ssize_t PyString_GET_SIZE(object o) except -1
37 object PyInt_FromString(char *str, char **pend, int base)
38 int Py_GetRecursionLimit()62 int Py_GetRecursionLimit()
39 int Py_EnterRecursiveCall(char *)63 int Py_EnterRecursiveCall(char *)
40 void Py_LeaveRecursiveCall()64 void Py_LeaveRecursiveCall()
4165
42 int PyList_Append(object, object) except -1
43
44cdef extern from "stdlib.h":
45 void free(void *memblock)
46 void *malloc(size_t size)
47 void *realloc(void *memblock, size_t size)
48 long strtol(char *, char **, int)
49
50cdef extern from "string.h":
51 void *memcpy(void *dest, void *src, size_t count)
52
53cdef extern from "python-compat.h":66cdef extern from "python-compat.h":
54 int snprintf(char* buffer, size_t nsize, char* fmt, ...)67 int snprintf(char* buffer, size_t nsize, char* fmt, ...)
5568
@@ -78,12 +91,12 @@
78 """Initialize decoder engine.91 """Initialize decoder engine.
79 @param s: Python string.92 @param s: Python string.
80 """93 """
81 if not PyString_CheckExact(s):94 if not PyBytes_CheckExact(s):
82 raise TypeError("String required")95 raise TypeError("bytes required")
8396
84 self.text = s97 self.text = s
85 self.tail = PyString_AS_STRING(s)98 self.tail = PyBytes_AS_STRING(s)
86 self.size = PyString_GET_SIZE(s)99 self.size = PyBytes_GET_SIZE(s)
87 self._yield_tuples = int(yield_tuples)100 self._yield_tuples = int(yield_tuples)
88101
89 def decode(self):102 def decode(self):
@@ -166,13 +179,13 @@
166 raise ValueError('leading zeros are not allowed')179 raise ValueError('leading zeros are not allowed')
167 D_UPDATE_TAIL(self, next_tail - self.tail + 1)180 D_UPDATE_TAIL(self, next_tail - self.tail + 1)
168 if n == 0:181 if n == 0:
169 return ''182 return b''
170 if n > self.size:183 if n > self.size:
171 raise ValueError('stream underflow')184 raise ValueError('stream underflow')
172 if n < 0:185 if n < 0:
173 raise ValueError('string size below zero: %d' % n)186 raise ValueError('string size below zero: %d' % n)
174187
175 result = PyString_FromStringAndSize(self.tail, n)188 result = PyBytes_FromStringAndSize(self.tail, n)
176 D_UPDATE_TAIL(self, n)189 D_UPDATE_TAIL(self, n)
177 return result190 return result
178191
@@ -210,7 +223,7 @@
210 if self.tail[0] < c'0' or self.tail[0] > c'9':223 if self.tail[0] < c'0' or self.tail[0] > c'9':
211 raise ValueError('key was not a simple string.')224 raise ValueError('key was not a simple string.')
212 key = self._decode_string()225 key = self._decode_string()
213 if lastkey >= key:226 if lastkey is not None and lastkey >= key:
214 raise ValueError('dict keys disordered')227 raise ValueError('dict keys disordered')
215 else:228 else:
216 lastkey = key229 lastkey = key
@@ -260,7 +273,7 @@
260 self.size = 0273 self.size = 0
261 self.tail = NULL274 self.tail = NULL
262275
263 p = <char*>malloc(maxsize)276 p = <char*>PyMem_Malloc(maxsize)
264 if p == NULL:277 if p == NULL:
265 raise MemoryError('Not enough memory to allocate buffer '278 raise MemoryError('Not enough memory to allocate buffer '
266 'for encoder')279 'for encoder')
@@ -269,15 +282,14 @@
269 self.tail = p282 self.tail = p
270283
271 def __dealloc__(self):284 def __dealloc__(self):
272 free(self.buffer)285 PyMem_Free(self.buffer)
273 self.buffer = NULL286 self.buffer = NULL
274 self.maxsize = 0287 self.maxsize = 0
275288
276 def __str__(self):289 def to_bytes(self):
277 if self.buffer != NULL and self.size != 0:290 if self.buffer != NULL and self.size != 0:
278 return PyString_FromStringAndSize(self.buffer, self.size)291 return PyBytes_FromStringAndSize(self.buffer, self.size)
279 else:292 return b''
280 return ''
281293
282 cdef int _ensure_buffer(self, int required) except 0:294 cdef int _ensure_buffer(self, int required) except 0:
283 """Ensure that tail of CharTail buffer has enough size.295 """Ensure that tail of CharTail buffer has enough size.
@@ -293,7 +305,7 @@
293 new_size = self.maxsize305 new_size = self.maxsize
294 while new_size < self.size + required:306 while new_size < self.size + required:
295 new_size = new_size * 2307 new_size = new_size * 2
296 new_buffer = <char*>realloc(self.buffer, <size_t>new_size)308 new_buffer = <char*>PyMem_Realloc(self.buffer, <size_t>new_size)
297 if new_buffer == NULL:309 if new_buffer == NULL:
298 raise MemoryError('Cannot realloc buffer for encoder')310 raise MemoryError('Cannot realloc buffer for encoder')
299311
@@ -308,32 +320,32 @@
308 """320 """
309 cdef int n321 cdef int n
310 self._ensure_buffer(INT_BUF_SIZE)322 self._ensure_buffer(INT_BUF_SIZE)
311 n = snprintf(self.tail, INT_BUF_SIZE, "i%de", x)323 n = snprintf(self.tail, INT_BUF_SIZE, b"i%de", x)
312 if n < 0:324 if n < 0:
313 raise MemoryError('int %d too big to encode' % x)325 raise MemoryError('int %d too big to encode' % x)
314 E_UPDATE_TAIL(self, n)326 E_UPDATE_TAIL(self, n)
315 return 1327 return 1
316328
317 cdef int _encode_long(self, x) except 0:329 cdef int _encode_long(self, x) except 0:
318 return self._append_string(''.join(('i', str(x), 'e')))330 return self._append_string(b'i%de' % x)
319331
320 cdef int _append_string(self, s) except 0:332 cdef int _append_string(self, s) except 0:
321 cdef Py_ssize_t n333 cdef Py_ssize_t n
322 n = PyString_GET_SIZE(s)334 n = PyBytes_GET_SIZE(s)
323 self._ensure_buffer(n)335 self._ensure_buffer(n)
324 memcpy(self.tail, PyString_AS_STRING(s), n)336 memcpy(self.tail, PyBytes_AS_STRING(s), n)
325 E_UPDATE_TAIL(self, n)337 E_UPDATE_TAIL(self, n)
326 return 1338 return 1
327339
328 cdef int _encode_string(self, x) except 0:340 cdef int _encode_string(self, x) except 0:
329 cdef int n341 cdef int n
330 cdef Py_ssize_t x_len342 cdef Py_ssize_t x_len
331 x_len = PyString_GET_SIZE(x)343 x_len = PyBytes_GET_SIZE(x)
332 self._ensure_buffer(x_len + INT_BUF_SIZE)344 self._ensure_buffer(x_len + INT_BUF_SIZE)
333 n = snprintf(self.tail, INT_BUF_SIZE, '%d:', x_len)345 n = snprintf(self.tail, INT_BUF_SIZE, b'%d:', x_len)
334 if n < 0:346 if n < 0:
335 raise MemoryError('string %s too big to encode' % x)347 raise MemoryError('string %s too big to encode' % x)
336 memcpy(<void *>(self.tail+n), PyString_AS_STRING(x), x_len)348 memcpy(<void *>(self.tail+n), PyBytes_AS_STRING(x), x_len)
337 E_UPDATE_TAIL(self, n + x_len)349 E_UPDATE_TAIL(self, n + x_len)
338 return 1350 return 1
339351
@@ -355,10 +367,8 @@
355 self.tail[0] = c'd'367 self.tail[0] = c'd'
356 E_UPDATE_TAIL(self, 1)368 E_UPDATE_TAIL(self, 1)
357369
358 keys = x.keys()370 for k in sorted(x):
359 keys.sort()371 if not PyBytes_CheckExact(k):
360 for k in keys:
361 if not PyString_CheckExact(k):
362 raise TypeError('key in dict should be string')372 raise TypeError('key in dict should be string')
363 self._encode_string(k)373 self._encode_string(k)
364 self.process(x[k])374 self.process(x[k])
@@ -372,14 +382,14 @@
372 if Py_EnterRecursiveCall("encode"):382 if Py_EnterRecursiveCall("encode"):
373 raise RuntimeError("too deeply nested")383 raise RuntimeError("too deeply nested")
374 try:384 try:
375 if PyString_CheckExact(x):385 if PyBytes_CheckExact(x):
376 self._encode_string(x)386 self._encode_string(x)
377 elif PyInt_CheckExact(x):387 elif PyInt_CheckExact(x) and x.bit_length() < 32:
378 self._encode_int(x)388 self._encode_int(x)
379 elif PyLong_CheckExact(x):389 elif PyLong_CheckExact(x):
380 self._encode_long(x)390 self._encode_long(x)
381 elif (PyList_CheckExact(x) or PyTuple_CheckExact(x)391 elif (PyList_CheckExact(x) or PyTuple_CheckExact(x)
382 or StaticTuple_CheckExact(x)):392 or isinstance(x, StaticTuple)):
383 self._encode_list(x)393 self._encode_list(x)
384 elif PyDict_CheckExact(x):394 elif PyDict_CheckExact(x):
385 self._encode_dict(x)395 self._encode_dict(x)
@@ -397,4 +407,4 @@
397 """Encode Python object x to string"""407 """Encode Python object x to string"""
398 encoder = Encoder()408 encoder = Encoder()
399 encoder.process(x)409 encoder.process(x)
400 return str(encoder)410 return encoder.to_bytes()

Subscribers

People subscribed via source and target branches