Merge lp:~nshepperd/apt-zeroconf/parse-hacking into lp:apt-zeroconf

Proposed by Neil Shepperd on 2009-08-30
Status: Needs review
Proposed branch: lp:~nshepperd/apt-zeroconf/parse-hacking
Merge into: lp:apt-zeroconf
Diff against target: None lines
To merge this branch: bzr merge lp:~nshepperd/apt-zeroconf/parse-hacking
Reviewer Review Type Date Requested Status
Apt Zeroconf Team 2009-08-30 Pending
Review via email: mp+10888@code.launchpad.net
To post a comment you must log in.
Neil Shepperd (nshepperd) wrote :

I fixed a bug preventing apt-zeroconf from downloading locally packages with an epoch in the version. The problem involved http unquoting ("%3a" -> ":") of package names in the list, and requoting, which changed the quoted part to uppercase. I felt it was unsafe to do this sort of manipulation in the package list anyway, so I removed that quote/unquoting.

I also changed a bit of the way the list parser works, mostly because I think it makes it more readable. However that is subjective, of course.

I'd like some feedback on these changes, please, and if everyone thinks they're okay, I'll merge them.

Neil Shepperd (nshepperd) wrote :

Eek, launchpad doesn't seem to be line-wrapping my summary... posted again:

> I fixed a bug preventing apt-zeroconf from downloading locally packages with
> an epoch in the version. The problem involved http unquoting ("%3a" -> ":") of
> package names in the list, and requoting, which changed the quoted part to
> uppercase. I felt it was unsafe to do this sort of manipulation in the package
> list anyway, so I removed that quote/unquoting.
>
> I also changed a bit of the way the list parser works, mostly because I think
> it makes it more readable. However that is subjective, of course.
>
> I'd like some feedback on these changes, please, and if everyone thinks
> they're okay, I'll merge them.

128. By Neil Shepperd on 2009-09-12

Merge back from trunk.

Unmerged revisions

128. By Neil Shepperd on 2009-09-12

Merge back from trunk.

127. By Neil Shepperd on 2009-08-09

Remove unused 'V3' epoch pattern regular expression.

126. By Neil Shepperd on 2009-08-09

Add test for downloading a deb with an epoch, which the parse.py changes
probably broke, and add fixes so that test passes.

125. By Neil Shepperd on 2009-08-09

Major disruption of parse.py to simplify extremely.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'aptzeroconf/parse.py'
2--- aptzeroconf/parse.py 2009-08-09 02:39:51 +0000
3+++ aptzeroconf/parse.py 2009-08-09 04:00:35 +0000
4@@ -21,116 +21,70 @@
5 from xml.sax import parseString, ContentHandler
6 #from xml.sax.handler import feature_namespaces
7 from xml.sax._exceptions import SAXParseException
8+from apt_pkg import VersionCompare
9 import logging
10 import re
11
12 __version__ = float(0.3)
13
14 V2, V3 = 0.2, 0.3
15-EPOCH_PATTERN = {
16- V2 : re.compile(r'\_\d+\%3a'),
17- V3 : re.compile(r'\_\d+\:')
18-}
19+EPOCH_PATTERN = re.compile(r'\_\d+\%3a')
20
21 def normalize_whitespace(text):
22 """Remove excessive whitespace in spaces."""
23 return ' '.join(text.split())
24
25-def _is_equal(pkg, find, ver):
26+def _is_equal(pkg, find):
27 """Are the packages equal disregarding the epoch?"""
28- return EPOCH_PATTERN[ver].sub('_', pkg) == find or pkg == find
29-
30-
31-class _PackageList(list):
32- """
33- Found Package list container.
34- """
35-
36- def found(self, item):
37- """
38- Callback to add a found package to contatiner
39-
40- @param item: Package Name
41- @type item: basestring
42- """
43- self.append(item)
44-
45- def __str__(self):
46- """
47- String representaion of found package container. Displays the best
48- package found in container if multiples were added.
49-
50- @return: Best or only package
51- @rtype: basestring
52- """
53- from apt_pkg import VersionCompare
54- from urllib2 import quote
55- if len(self) > 1:
56- self.sort(cmp=VersionCompare)
57- return quote(self[-1])
58-
59- elif len(self) == 1:
60- return quote(self[0])
61-
62- else:
63- return ""
64-
65+ return EPOCH_PATTERN.sub('_', pkg) == find or pkg == find
66+
67+
68+class Package(object):
69+ """Represents a single package."""
70+ def __init__(self, name=''):
71+ self.name = name
72 def __repr__(self):
73- return '<Package name="%s" count="%d">' % (str(self), len(self))
74-
75-
76-class _PackageFinder(ContentHandler):
77+ return '<Package name=' + self.name + '>'
78+
79+
80+class _XMLDecoder(ContentHandler):
81+ """Decodes an xml package list, calling a callback for each package."""
82 callback = None #: Callback function to be set when item found
83- inPkgElemCnt = {}
84
85- def __init__(self, package):
86+ def __init__(self):
87 """
88 Initialize sax content handler.
89
90 @param search_name: Package to locate in search data.
91 @type search_name: basestring
92 """
93- self.packageSearch = normalize_whitespace(package)
94-
95- self.inPkgName = False #: Initialize the content flag to False
96- self.inPkg = False #: Initialize the content flag to False
97- self.list_version = '' #: Initialize the version value to None
98+ self.text = ''
99+ self.package = None
100+ self.list_version = ''
101
102 def startElement(self, name, attrs):
103 """
104 Check start element tags for packagelist or name.
105
106- @param name: element tag.
107- @type name: basestring
108- @param name: element tag attrs.
109- @type name: basestring
110+ @param name: element tag
111+ @type name: basestring
112+ @param name: element attributes
113+ @type name: dict
114 """
115 if name == 'packagelist': #: handle element by getting ver. attribute
116 self.list_version = normalize_whitespace(attrs.get('version', ""))
117-
118- if float(self.list_version) == __version__:
119-
120- if name == 'package':
121- self.inPkg = True
122- self.inPkgElemCnt = {'name': 0}
123-
124- elif name == 'name': #: handle element by set flag, reset value
125- if self.inPkgElemCnt[name] > 1:
126- raise SAXParseException
127-
128- self.inPkgName = True
129- self.packageName = ""
130- self.inPkgElemCnt[name] += 1
131+ if name == 'package':
132+ self.package = Package()
133+ self.text = ''
134
135 def characters(self, char):
136 """
137- Check characters for name value.
138+ Add character input to text buffer.
139
140 @param char: text chunk
141 @type name: basestring
142 """
143- if self.inPkgName:
144- self.packageName = self.packageName + char
145+ self.text += char
146
147 def endElement(self, name):
148 """
149@@ -140,70 +94,71 @@
150 @param name: element tag.
151 @type name: basestring
152 """
153- if float(self.list_version) == __version__:
154- if name == 'package':
155- self.inPkgName = False
156- elif name == 'name':
157- self.inPkgName = False
158- self.packageName = normalize_whitespace(self.packageName)
159-
160- #{ Check search_name for match with packagename
161- if _is_equal(self.packageName, self.packageSearch, V3):
162- self.callback(self.packageName)
163-
164-
165-def has_package(package, data):
166+ if name == 'package':
167+ self.callback(self.package)
168+ elif name == 'name':
169+ self.package.name = self.text
170+
171+def decode_list(data):
172 """
173- Locate package in data string.
174-
175- @param package: package name
176- @type package: basestring
177- @param data: xml package data
178+ @param data: xml/raw package data
179 @type data: basestring
180- @return: PkgList object
181- @rtype: object
182+ @return: list of Packages
183+ @rtype: list
184 """
185- pkglist = _PackageList()
186 if not data.startswith('<?xml'):
187- for name in data.split():
188- if _is_equal(name, package, V2):
189- pkglist.found(name)
190- return pkglist
191+ data = data.split()
192+ if data[0] == str(V2):
193+ return [Package(name) for name in data[1:]]
194
195- handler = _PackageFinder(package)
196- handler.callback = pkglist.found
197+ pkglist = []
198+ handler = _XMLDecoder()
199+ handler.callback = pkglist.append
200
201 try:
202 parseString(data, handler)
203-
204 except SAXParseException:
205 #{ Non-conforming list
206 logging.warning(
207 '[PARSER] Parse Error, Ignoring: malformed package list.')
208- return _PackageList()
209-
210+ return []
211 return pkglist
212
213
214+def has_package(package, list):
215+ """
216+ Locate package (disregarding the epoch) in package list.
217+
218+ Returns the package name, including the epoch if there is one.
219+
220+ @param package: package name
221+ @type package: basestring
222+ @param data: list of Packages
223+ @type data: list
224+ @return: package name
225+ @rtype: basestring
226+ """
227+ results = []
228+ for pkg in list:
229+ if _is_equal(pkg.name, package):
230+ results.append(pkg.name)
231+
232+ results.sort(cmp=VersionCompare)
233+ if results:
234+ return results[-1]
235+ else:
236+ return ''
237+
238+
239 def store_packages(url, port):
240 """
241 Fetch non-local AZC Cache service host cache listings.
242-
243- @param url:
244- @type url:
245 """
246 from urllib2 import Request, urlopen
247
248- pkglist = urlopen(Request(url="http://%s:%d/list" % (url, port)))
249- pkgdata = pkglist.read(4)
250-
251- if pkgdata[:3] == V2:
252- return pkglist.read()
253-
254- elif pkgdata == '<?xm':
255- return pkgdata + pkglist.read()
256-
257- return ''
258+ pkglist = urlopen("http://%s:%d/list" % (url, port))
259+ data = pkglist.read()
260+ return decode_list(data)
261
262
263 def make_packagelist(packages):
264@@ -215,12 +170,10 @@
265 @return: xml package data
266 @rtype: basestring
267 """
268- from urllib2 import unquote
269-
270 form = ("<?xml version='1.0' ?>\n"
271 "<?xml-stylesheet href='list.xsl' type='text/xsl' ?>\n"
272 "<packagelist version='%.1f'>\n" % __version__)
273 for pkg in packages:
274- form += " <package>\n <name>%s</name>\n </package>" % unquote(pkg)
275+ form += " <package>\n <name>%s</name>\n </package>" % pkg
276 form += "</packagelist>\n"
277 return form
278
279=== modified file 'test/test_general.py'
280--- test/test_general.py 2009-07-24 23:24:41 +0000
281+++ test/test_general.py 2009-08-09 03:52:46 +0000
282@@ -124,6 +124,33 @@
283 PORT, DEBNAME)).read()
284 # make sure the data is equal
285 assert data == '*contents*'
286+
287+def test_download_epoch_local_server(tmpdir):
288+ """
289+ Test fetching a file with an epoch from an AZC instance on the local network.
290+ """
291+ # cachedir is the directory to serve debs out of
292+ one = path.join(str(tmpdir), 'one')
293+ two = path.join(str(tmpdir), 'two')
294+ cacheone = path.join(one, 'debs')
295+ cachetwo = path.join(two, 'debs')
296+ os.mkdir(one)
297+ os.mkdir(two)
298+ os.mkdir(cacheone)
299+ os.mkdir(cachetwo)
300+ touch(path.join(cachetwo, DEBEPOCHNAME), '*contents*')
301+
302+ one_config = {('apt-zeroconf', 'restrict'): 'Off',
303+ ('apt-zeroconf', 'hostname'): 'one'}
304+ two_config = {('apt-zeroconf', 'restrict'): 'Off',
305+ ('apt-zeroconf', 'hostname'): 'two'}
306+ with azc_server(one, cacheone, PORT, one_config):
307+ with azc_server(two, cachetwo, PORT+1, two_config):
308+ time.sleep(3.0) # wait for one to discover two
309+ data = urlopen('http://localhost:%i/http://example.com/%s' % (
310+ PORT, DEBNAME)).read()
311+ # make sure the data is equal
312+ assert data == '*contents*'
313
314
315 def test_download_proxy(tmpdir):

Subscribers

People subscribed via source and target branches