Merge lp:~gz/brz/py3_bencode_pyx into lp:brz

Proposed by Martin Packman on 2017-06-27
Status: Merged
Approved by: Martin Packman on 2017-06-27
Approved revision: 6725
Merge reported by: The Breezy Bot
Merged at revision: not available
Proposed branch: lp:~gz/brz/py3_bencode_pyx
Merge into: lp:brz
Prerequisite: lp:~gz/brz/py3_static_tuple_import
Diff against target: 237 lines (+64/-54)
1 file modified
breezy/_bencode_pyx.pyx (+64/-54)
To merge this branch: bzr merge lp:~gz/brz/py3_bencode_pyx
Reviewer Review Type Date Requested Status
Jelmer Vernooij 2017-06-27 Approve on 2017-06-27
Review via email: mp+326331@code.launchpad.net

Commit message

Make _bencode_pyx compile and pass tests on Python 3

Description of the change

A bunch of changes to modern cython style, which gets us free aliasing of the PyBytes names and such like.

One of the few tricky bits is the Int/Long switching, which should be sane. On encode, it's just a fast path, so now uses bit_length() to check the size rather than just inferring from type. (Trivia, bit_length doesn't do 2's compliment.) On decode PyInt_FromString will promote to long on Python 2, and is always long on 3.

To post a comment you must log in.
Jelmer Vernooij (jelmer) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'breezy/_bencode_pyx.pyx'
2--- breezy/_bencode_pyx.pyx 2017-06-10 01:43:31 +0000
3+++ breezy/_bencode_pyx.pyx 2017-06-27 01:35:34 +0000
4@@ -18,38 +18,51 @@
5
6 from __future__ import absolute_import
7
8+from cpython.bytes cimport (
9+ PyBytes_CheckExact,
10+ PyBytes_FromStringAndSize,
11+ PyBytes_AS_STRING,
12+ PyBytes_GET_SIZE,
13+ )
14+from cpython.long cimport (
15+ PyLong_CheckExact,
16+ )
17+from cpython.int cimport (
18+ PyInt_CheckExact,
19+ PyInt_FromString,
20+ )
21+from cpython.tuple cimport (
22+ PyTuple_CheckExact,
23+ )
24+from cpython.list cimport (
25+ PyList_CheckExact,
26+ PyList_Append,
27+ )
28+from cpython.dict cimport (
29+ PyDict_CheckExact,
30+ )
31+from cpython.bool cimport (
32+ PyBool_Check,
33+ )
34+from cpython.mem cimport (
35+ PyMem_Free,
36+ PyMem_Malloc,
37+ PyMem_Realloc,
38+ )
39
40-cdef extern from "stddef.h":
41- ctypedef unsigned int size_t
42+from libc.stdlib cimport (
43+ strtol,
44+ )
45+from libc.string cimport (
46+ memcpy,
47+ )
48
49 cdef extern from "Python.h":
50- ctypedef int Py_ssize_t
51- int PyInt_CheckExact(object o)
52- int PyLong_CheckExact(object o)
53- int PyString_CheckExact(object o)
54- int PyTuple_CheckExact(object o)
55- int PyList_CheckExact(object o)
56- int PyDict_CheckExact(object o)
57- int PyBool_Check(object o)
58- object PyString_FromStringAndSize(char *v, Py_ssize_t len)
59- char *PyString_AS_STRING(object o) except NULL
60- Py_ssize_t PyString_GET_SIZE(object o) except -1
61- object PyInt_FromString(char *str, char **pend, int base)
62+ # There is no cython module for ceval.h for some reason
63 int Py_GetRecursionLimit()
64 int Py_EnterRecursiveCall(char *)
65 void Py_LeaveRecursiveCall()
66
67- int PyList_Append(object, object) except -1
68-
69-cdef extern from "stdlib.h":
70- void free(void *memblock)
71- void *malloc(size_t size)
72- void *realloc(void *memblock, size_t size)
73- long strtol(char *, char **, int)
74-
75-cdef extern from "string.h":
76- void *memcpy(void *dest, void *src, size_t count)
77-
78 cdef extern from "python-compat.h":
79 int snprintf(char* buffer, size_t nsize, char* fmt, ...)
80
81@@ -78,12 +91,12 @@
82 """Initialize decoder engine.
83 @param s: Python string.
84 """
85- if not PyString_CheckExact(s):
86- raise TypeError("String required")
87+ if not PyBytes_CheckExact(s):
88+ raise TypeError("bytes required")
89
90 self.text = s
91- self.tail = PyString_AS_STRING(s)
92- self.size = PyString_GET_SIZE(s)
93+ self.tail = PyBytes_AS_STRING(s)
94+ self.size = PyBytes_GET_SIZE(s)
95 self._yield_tuples = int(yield_tuples)
96
97 def decode(self):
98@@ -166,13 +179,13 @@
99 raise ValueError('leading zeros are not allowed')
100 D_UPDATE_TAIL(self, next_tail - self.tail + 1)
101 if n == 0:
102- return ''
103+ return b''
104 if n > self.size:
105 raise ValueError('stream underflow')
106 if n < 0:
107 raise ValueError('string size below zero: %d' % n)
108
109- result = PyString_FromStringAndSize(self.tail, n)
110+ result = PyBytes_FromStringAndSize(self.tail, n)
111 D_UPDATE_TAIL(self, n)
112 return result
113
114@@ -210,7 +223,7 @@
115 if self.tail[0] < c'0' or self.tail[0] > c'9':
116 raise ValueError('key was not a simple string.')
117 key = self._decode_string()
118- if lastkey >= key:
119+ if lastkey is not None and lastkey >= key:
120 raise ValueError('dict keys disordered')
121 else:
122 lastkey = key
123@@ -260,7 +273,7 @@
124 self.size = 0
125 self.tail = NULL
126
127- p = <char*>malloc(maxsize)
128+ p = <char*>PyMem_Malloc(maxsize)
129 if p == NULL:
130 raise MemoryError('Not enough memory to allocate buffer '
131 'for encoder')
132@@ -269,15 +282,14 @@
133 self.tail = p
134
135 def __dealloc__(self):
136- free(self.buffer)
137+ PyMem_Free(self.buffer)
138 self.buffer = NULL
139 self.maxsize = 0
140
141- def __str__(self):
142+ def to_bytes(self):
143 if self.buffer != NULL and self.size != 0:
144- return PyString_FromStringAndSize(self.buffer, self.size)
145- else:
146- return ''
147+ return PyBytes_FromStringAndSize(self.buffer, self.size)
148+ return b''
149
150 cdef int _ensure_buffer(self, int required) except 0:
151 """Ensure that tail of CharTail buffer has enough size.
152@@ -293,7 +305,7 @@
153 new_size = self.maxsize
154 while new_size < self.size + required:
155 new_size = new_size * 2
156- new_buffer = <char*>realloc(self.buffer, <size_t>new_size)
157+ new_buffer = <char*>PyMem_Realloc(self.buffer, <size_t>new_size)
158 if new_buffer == NULL:
159 raise MemoryError('Cannot realloc buffer for encoder')
160
161@@ -308,32 +320,32 @@
162 """
163 cdef int n
164 self._ensure_buffer(INT_BUF_SIZE)
165- n = snprintf(self.tail, INT_BUF_SIZE, "i%de", x)
166+ n = snprintf(self.tail, INT_BUF_SIZE, b"i%de", x)
167 if n < 0:
168 raise MemoryError('int %d too big to encode' % x)
169 E_UPDATE_TAIL(self, n)
170 return 1
171
172 cdef int _encode_long(self, x) except 0:
173- return self._append_string(''.join(('i', str(x), 'e')))
174+ return self._append_string(b'i%de' % x)
175
176 cdef int _append_string(self, s) except 0:
177 cdef Py_ssize_t n
178- n = PyString_GET_SIZE(s)
179+ n = PyBytes_GET_SIZE(s)
180 self._ensure_buffer(n)
181- memcpy(self.tail, PyString_AS_STRING(s), n)
182+ memcpy(self.tail, PyBytes_AS_STRING(s), n)
183 E_UPDATE_TAIL(self, n)
184 return 1
185
186 cdef int _encode_string(self, x) except 0:
187 cdef int n
188 cdef Py_ssize_t x_len
189- x_len = PyString_GET_SIZE(x)
190+ x_len = PyBytes_GET_SIZE(x)
191 self._ensure_buffer(x_len + INT_BUF_SIZE)
192- n = snprintf(self.tail, INT_BUF_SIZE, '%d:', x_len)
193+ n = snprintf(self.tail, INT_BUF_SIZE, b'%d:', x_len)
194 if n < 0:
195 raise MemoryError('string %s too big to encode' % x)
196- memcpy(<void *>(self.tail+n), PyString_AS_STRING(x), x_len)
197+ memcpy(<void *>(self.tail+n), PyBytes_AS_STRING(x), x_len)
198 E_UPDATE_TAIL(self, n + x_len)
199 return 1
200
201@@ -355,10 +367,8 @@
202 self.tail[0] = c'd'
203 E_UPDATE_TAIL(self, 1)
204
205- keys = x.keys()
206- keys.sort()
207- for k in keys:
208- if not PyString_CheckExact(k):
209+ for k in sorted(x):
210+ if not PyBytes_CheckExact(k):
211 raise TypeError('key in dict should be string')
212 self._encode_string(k)
213 self.process(x[k])
214@@ -372,14 +382,14 @@
215 if Py_EnterRecursiveCall("encode"):
216 raise RuntimeError("too deeply nested")
217 try:
218- if PyString_CheckExact(x):
219+ if PyBytes_CheckExact(x):
220 self._encode_string(x)
221- elif PyInt_CheckExact(x):
222+ elif PyInt_CheckExact(x) and x.bit_length() < 32:
223 self._encode_int(x)
224 elif PyLong_CheckExact(x):
225 self._encode_long(x)
226 elif (PyList_CheckExact(x) or PyTuple_CheckExact(x)
227- or StaticTuple_CheckExact(x)):
228+ or isinstance(x, StaticTuple)):
229 self._encode_list(x)
230 elif PyDict_CheckExact(x):
231 self._encode_dict(x)
232@@ -397,4 +407,4 @@
233 """Encode Python object x to string"""
234 encoder = Encoder()
235 encoder.process(x)
236- return str(encoder)
237+ return encoder.to_bytes()

Subscribers

People subscribed via source and target branches