Merge lp:~jerith/ibid/split_utils into lp:~ibid-core/ibid/old-trunk-1.6

[Jeremy Thurgood]

[Jeremy Thurgood (jerith)]

A small amount of reorganisation makes html5lib and BeautifulSoup optional dependencies.

[Jonathan Hitchcock (vhata)]

ImportError: No module named ibid.utils.html

review: Needs Fixing

[Michael Gorven (mgorven)]

You forgot to add ibid/utils/html.py.
 review needs_fixing

review: Needs Fixing

[Michael Gorven (mgorven)]

  File "/home/mgorven/bzr/ibid/jerith/split_utils/ibid/utils/html.py", line 7,
in get_html_parse_tree
    req = urllib2.Request(url, data, headers)
NameError: global name 'urllib2' is not defined

 review needs_fixing

review: Needs Fixing

[lp:~jerith/ibid/split_utils]

810. By Jeremy Thurgood on 2009-12-28

Fixed broken imports.

Unmerged revisions

=== modified file 'ibid/plugins/feeds.py'
--- ibid/plugins/feeds.py	2009-12-10 09:56:44 +0000
+++ ibid/plugins/feeds.py	2009-12-28 13:53:13 +0000
@@ -12,7 +12,8 @@
 from ibid.config import IntOption
 from ibid.plugins import Processor, match, authorise, run_every
 from ibid.models import Base, VersionedSchema
-from ibid.utils import cacheable_download, get_html_parse_tree, human_join
+from ibid.utils import cacheable_download, human_join
+from ibid.utils.html import get_html_parse_tree
 
 help = {'feeds': u'Displays articles from RSS and Atom feeds'}
 

=== modified file 'ibid/plugins/icecast.py'
--- ibid/plugins/icecast.py	2009-11-30 15:29:43 +0000
+++ ibid/plugins/icecast.py	2009-12-28 13:53:13 +0000
@@ -3,7 +3,8 @@
 
 from ibid.config import DictOption, IntOption
 from ibid.plugins import Processor, match, run_every
-from ibid.utils import get_html_parse_tree, human_join
+from ibid.utils import human_join
+from ibid.utils.html import get_html_parse_tree
 
 log = logging.getLogger('plugins.icecast')
 

=== modified file 'ibid/plugins/lookup.py'
--- ibid/plugins/lookup.py	2009-12-17 16:22:53 +0000
+++ ibid/plugins/lookup.py	2009-12-28 13:53:13 +0000
@@ -15,8 +15,9 @@
 from ibid.compat import defaultdict, dt_strptime, ElementTree
 from ibid.config import Option, BoolOption, DictOption
 from ibid.plugins import Processor, match, handler
-from ibid.utils import ago, decode_htmlentities, get_html_parse_tree, \
-        cacheable_download, json_webservice, human_join, plural
+from ibid.utils import ago, decode_htmlentities, cacheable_download, \
+        json_webservice, human_join, plural
+from ibid.utils.html import get_html_parse_tree
 
 log = logging.getLogger('plugins.lookup')
 

=== modified file 'ibid/plugins/url.py'
--- ibid/plugins/url.py	2009-10-18 16:49:32 +0000
+++ ibid/plugins/url.py	2009-12-28 13:53:13 +0000
@@ -13,7 +13,7 @@
 from ibid.plugins import Processor, match, handler
 from ibid.config import Option, ListOption
 from ibid.models import Base, VersionedSchema
-from ibid.utils  import get_html_parse_tree
+from ibid.utils.html import get_html_parse_tree
 
 help = {'url': u'Captures URLs seen in channel to database and/or to delicious, and shortens and lengthens URLs'}
 

=== added directory 'ibid/utils'
=== renamed file 'ibid/utils.py' => 'ibid/utils/__init__.py'
--- ibid/utils.py	2009-12-20 22:09:29 +0000
+++ ibid/utils/__init__.py	2009-12-28 13:53:13 +0000
@@ -1,4 +1,3 @@
-import cgi
 from gzip import GzipFile
 from htmlentitydefs import name2codepoint
 import os
@@ -12,8 +11,6 @@
 import zlib
 
 from dateutil.tz import tzlocal, tzutc
-from html5lib import HTMLParser, treebuilders
-from BeautifulSoup import BeautifulSoup
 
 import ibid
 from ibid.compat import defaultdict, ElementTree, json
@@ -161,48 +158,6 @@
 class ContentTypeException(Exception):
     pass
 
-def get_html_parse_tree(url, data=None, headers={}, treetype='beautifulsoup'):
-    "Request a URL, parse with html5lib, and return a parse tree from it"
-
-    req = urllib2.Request(url, data, headers)
-    f = urllib2.urlopen(req)
-
-    if f.info().gettype() not in ('text/html', 'application/xhtml+xml'):
-        f.close()
-        raise ContentTypeException("Content type isn't HTML, but " + f.info().gettype())
-
-    data = f.read()
-    f.close()
-
-    encoding = None
-    contentType = f.headers.get('content-type')
-    if contentType:
-        (mediaType, params) = cgi.parse_header(contentType)
-        encoding = params.get('charset')
-
-    compression = f.headers.get('content-encoding')
-    if compression:
-        if compression.lower() == "deflate":
-            try:
-                data = zlib.decompress(data)
-            except zlib.error:
-                data = zlib.decompress(data, -zlib.MAX_WBITS)
-        elif compression.lower() == "gzip":
-            compressedstream = StringIO(data)
-            gzipper = GzipFile(fileobj=compressedstream)
-            data = gzipper.read()
-
-    if treetype == "beautifulsoup":
-        return BeautifulSoup(data, convertEntities=BeautifulSoup.HTML_ENTITIES)
-    elif treetype == "etree":
-        treebuilder = treebuilders.getTreeBuilder("etree", ElementTree)
-    else:
-        treebuilder = treebuilders.getTreeBuilder(treetype)
-
-    parser = HTMLParser(tree=treebuilder)
-
-    return parser.parse(data, encoding = encoding)
-
 class JSONException(Exception):
     pass
 

=== added file 'ibid/utils/html.py'
--- ibid/utils/html.py	1970-01-01 00:00:00 +0000
+++ ibid/utils/html.py	2009-12-28 13:53:13 +0000
@@ -0,0 +1,44 @@
+from html5lib import HTMLParser, treebuilders
+from BeautifulSoup import BeautifulSoup
+
+def get_html_parse_tree(url, data=None, headers={}, treetype='beautifulsoup'):
+    "Request a URL, parse with html5lib, and return a parse tree from it"
+
+    req = urllib2.Request(url, data, headers)
+    f = urllib2.urlopen(req)
+
+    if f.info().gettype() not in ('text/html', 'application/xhtml+xml'):
+        f.close()
+        raise ContentTypeException("Content type isn't HTML, but " + f.info().gettype())
+
+    data = f.read()
+    f.close()
+
+    encoding = None
+    contentType = f.headers.get('content-type')
+    if contentType:
+        (mediaType, params) = cgi.parse_header(contentType)
+        encoding = params.get('charset')
+
+    compression = f.headers.get('content-encoding')
+    if compression:
+        if compression.lower() == "deflate":
+            try:
+                data = zlib.decompress(data)
+            except zlib.error:
+                data = zlib.decompress(data, -zlib.MAX_WBITS)
+        elif compression.lower() == "gzip":
+            compressedstream = StringIO(data)
+            gzipper = GzipFile(fileobj=compressedstream)
+            data = gzipper.read()
+
+    if treetype == "beautifulsoup":
+        return BeautifulSoup(data, convertEntities=BeautifulSoup.HTML_ENTITIES)
+    elif treetype == "etree":
+        treebuilder = treebuilders.getTreeBuilder("etree", ElementTree)
+    else:
+        treebuilder = treebuilders.getTreeBuilder(treetype)
+
+    parser = HTMLParser(tree=treebuilder)
+
+    return parser.parse(data, encoding = encoding)