Merge lp:~stefanor/ibid/url-terminology-528779 into lp:~ibid-core/ibid/old-trunk-1.6

[Stefano Rivera]

Rename ibid.utils.url_to_bytestring -> iri_to_uri and improve the accuracy of the conversion

[Max Rabkin (max-rabkin)]

review: Approve

[Jonathan Hitchcock (vhata)]

review: Approve

[Michael Gorven (mgorven)]

 review approve

review: Approve

=== modified file 'docs/api/ibid.utils.rst'
--- docs/api/ibid.utils.rst	2010-03-02 20:29:27 +0000
+++ docs/api/ibid.utils.rst	2010-03-15 21:08:14 +0000
@@ -122,10 +122,10 @@
 
    Is *url* a valid URL? (according to :func:`url_regex`)
 
-.. function:: url_to_bytestring(url)
+.. function:: iri_to_uri(iri)
 
-   Convert a unicode *url* to punycode host and UTF-8 path.
-   This allows IDN URLs to be opened with :mod:`urllib`.
+   Convert a unicode *iri* to punycode host and UTF-8 path.
+   This allows IRIs to be opened with :mod:`urllib`.
 
 Web Service Functions
 ---------------------

=== modified file 'ibid/plugins/ascii.py'
--- ibid/plugins/ascii.py	2010-02-20 21:58:38 +0000
+++ ibid/plugins/ascii.py	2010-03-15 21:08:14 +0000
@@ -17,7 +17,7 @@
 
 from ibid.config import Option, IntOption
 from ibid.plugins import Processor, match
-from ibid.utils import file_in_path, url_to_bytestring
+from ibid.utils import file_in_path, iri_to_uri
 
 """
 Dependencies:
@@ -60,7 +60,7 @@
             url += '/'
 
         try:
-            f = urlopen(url_to_bytestring(url))
+            f = urlopen(iri_to_uri(url))
         except HTTPError, e:
             event.addresponse(u'Sorry, error fetching URL: %s', BaseHTTPRequestHandler.responses[e.code][0])
             return

=== modified file 'ibid/plugins/languages.py'
--- ibid/plugins/languages.py	2010-03-02 19:41:09 +0000
+++ ibid/plugins/languages.py	2010-03-15 21:08:14 +0000
@@ -11,7 +11,7 @@
 from ibid.plugins import Processor, match
 from ibid.config import Option, IntOption
 from ibid.utils import decode_htmlentities, json_webservice, human_join, \
-                        is_url, url_to_bytestring
+                        is_url, iri_to_uri
 
 features = {}
 
@@ -192,7 +192,7 @@
 
         if is_url(text):
             if urlparse(text).scheme in ('', 'http'):
-                url = url_to_bytestring(text)
+                url = iri_to_uri(text)
                 query = {'sl': src_lang, 'tl': dest_lang, 'u': url}
                 event.addresponse(u'http://translate.google.com/translate?' +
                                     urlencode(query))

=== modified file 'ibid/plugins/network.py'
--- ibid/plugins/network.py	2010-03-07 10:08:03 +0000
+++ ibid/plugins/network.py	2010-03-15 21:08:14 +0000
@@ -22,7 +22,7 @@
 from ibid.plugins import Processor, match, authorise
 from ibid.config import Option, IntOption, FloatOption, DictOption
 from ibid.utils import file_in_path, get_country_codes, get_process_output, \
-                       human_join, unicode_output, url_to_bytestring
+                       human_join, unicode_output, iri_to_uri
 
 features = {}
 
@@ -391,7 +391,7 @@
 
         try:
             try:
-                conn.request(method.upper(), url_to_bytestring(url),
+                conn.request(method.upper(), iri_to_uri(url),
                              headers=headers)
                 response = conn.getresponse()
                 data = response.read(self.max_size)

=== modified file 'ibid/utils/__init__.py'
--- ibid/utils/__init__.py	2010-03-07 10:06:56 +0000
+++ ibid/utils/__init__.py	2010-03-15 21:08:14 +0000
@@ -84,7 +84,7 @@
 
     exists = os.path.isfile(cachefile)
 
-    req = urllib2.Request(url_to_bytestring(url))
+    req = urllib2.Request(iri_to_uri(url))
     for name, value in headers.iteritems():
         req.add_header(name, value)
     if not req.has_header('user-agent'):
@@ -190,12 +190,25 @@
 class JSONException(Exception):
     pass
 
-def url_to_bytestring(url):
+def iri_to_uri(url):
     "Expand an IDN hostname and UTF-8 encode the path of a unicode URL"
     parts = list(urlparse(url))
-    host = parts[1].split(':')
-    host[0] = host[0].encode('idna')
-    parts[1] = ':'.join(host)
+    username, passwd, host, port = re.match(
+        r'^(?:(.*)(?::(.*))?@)?(.*)(?::(.*))?$', parts[1]).groups()
+    parts[1] = ''
+    if username:
+        parts[1] = quote(username.encode('utf-8'))
+        if passwd:
+            parts[1] += ':' + quote(passwd.encode('utf-8'))
+        parts[1] += '@'
+    if host:
+        if parts[0].lower() in ('http', 'https', 'ftp'):
+            parts[1] += host.encode('idna')
+        else:
+            parts[1] += quote(host.encode('utf-8'))
+    if port:
+        parts[1] += ':' + quote(port.encode('utf-8'))
+
     parts[2] = quote(parts[2].encode('utf-8'), '/%')
     return urlunparse(parts).encode('utf-8')
 
@@ -234,7 +247,7 @@
             params[key] = params[key].encode('utf-8')
 
     if params:
-        url = url_to_bytestring(url) + '?' + urlencode(params)
+        url = iri_to_uri(url) + '?' + urlencode(params)
 
     req = urllib2.Request(url, headers=headers)
     if not req.has_header('user-agent'):

=== modified file 'ibid/utils/html.py'
--- ibid/utils/html.py	2010-03-07 09:42:37 +0000
+++ ibid/utils/html.py	2010-03-15 21:08:14 +0000
@@ -12,7 +12,7 @@
 from BeautifulSoup import BeautifulSoup
 
 from ibid.compat import ElementTree
-from ibid.utils import url_to_bytestring
+from ibid.utils import iri_to_uri
 
 class ContentTypeException(Exception):
     pass
@@ -20,7 +20,7 @@
 def get_html_parse_tree(url, data=None, headers={}, treetype='beautifulsoup'):
     "Request a URL, parse with html5lib, and return a parse tree from it"
 
-    req = urllib2.Request(url_to_bytestring(url), data, headers)
+    req = urllib2.Request(iri_to_uri(url), data, headers)
     f = urllib2.urlopen(req)
 
     if f.info().gettype() not in ('text/html', 'application/xhtml+xml'):