Merge ~cjwatson/launchpad:bs4-feeds into launchpad:master

[Colin Watson]

Port feed tests to Beautiful Soup 4

3535948... by Colin Watson on 2020-01-23

Port xx-announcements.txt too

It uses lp.services.feeds.tests.helper, so needs to be ported along with
the other feed tests.

By default, Beautiful Soup 4 produces output with HTML entities
converted to Unicode characters. I could have used formatter="html" or
similar to restore something closer to the old behaviour in this case,
but the new behaviour is easier to read.

[Colin Watson (cjwatson)]

Self-approving: this is almost entirely tests, and is mechanical enough that it's not very interesting to review.

review: Approve

diff --git a/lib/lp/bugs/stories/feeds/xx-bug-atom.txt b/lib/lp/bugs/stories/feeds/xx-bug-atom.txt
index 4b0faa4..840b943 100644
--- a/lib/lp/bugs/stories/feeds/xx-bug-atom.txt
+++ b/lib/lp/bugs/stories/feeds/xx-bug-atom.txt
@@ -1,10 +1,12 @@
 = Atom Feeds =
 
 Atom feeds produce XML not HTML.  Therefore we must parse the output as XML
-using BeautifulStoneSoup instead of BSS or the helper functions.
+by asking BeautifulSoup to use lxml.
 
-    >>> from BeautifulSoup import BeautifulStoneSoup as BSS
-    >>> from BeautifulSoup import SoupStrainer
+    >>> from lp.services.beautifulsoup import (
+    ...     BeautifulSoup4 as BeautifulSoup,
+    ...     SoupStrainer4 as SoupStrainer,
+    ...     )
     >>> from lp.services.feeds.tests.helper import (
     ...     parse_entries, parse_links, validate_feed)
 
@@ -26,25 +28,26 @@ point to the bugs themselves.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs in Jokosher']
     >>> browser.url
     'http://feeds.launchpad.test/jokosher/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2007-03-15:/bugs/jokosher
     >>> alternate_links = parse_links(browser.contents, 'alternate')
     >>> for link in alternate_links:
     ...     print(link)
-    <link rel="alternate" href="http://bugs.launchpad.test/jokosher" />
-    <link rel="alternate" href="http://bugs.launchpad.test/bugs/12" />
-    <link rel="alternate" href="http://bugs.launchpad.test/bugs/11" />
+    <link href="http://bugs.launchpad.test/jokosher" rel="alternate"/>
+    <link href="http://bugs.launchpad.test/bugs/12" rel="alternate"/>
+    <link href="http://bugs.launchpad.test/bugs/11" rel="alternate"/>
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/jokosher/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/jokosher/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -83,19 +86,20 @@ as the latest bugs feed for a product.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs in The Mozilla Project']
     >>> browser.url
     'http://feeds.launchpad.test/mozilla/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2004-09-24:/bugs/mozilla
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/mozilla/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/mozilla/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -144,19 +148,20 @@ of content as the latest bugs feed for a product.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs in Ubuntu']
     >>> browser.url
     'http://feeds.launchpad.test/ubuntu/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2006-10-16:/bugs/ubuntu
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/ubuntu/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/ubuntu/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -214,11 +219,8 @@ The bug should be included in the feed.
 
 Private teams should show as '-'.
 
-    >>> entry_content = BSS(
-    ...     entry.find('content').text,
-    ...     convertEntities=BSS.HTML_ENTITIES)
-    >>> soup = BSS(entry_content.text)
-    >>> print([tr.findAll('td')[4].text for tr in soup.findAll('tr')[1:4]])
+    >>> soup = BeautifulSoup(entry.find('content').text, 'xml')
+    >>> print([tr.find_all('td')[4].text for tr in soup.find_all('tr')[1:4]])
     [u'Mark Shuttleworth', u'-', u'-']
 
 == Latest bugs for a source package ==
@@ -232,11 +234,12 @@ type of content as the latest bugs feed for a product.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs in thunderbird in Ubuntu']
     >>> browser.url
     'http://feeds.launchpad.test/ubuntu/+source/thunderbird/latest-bugs.atom'
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2008:/bugs/ubuntu/+source/thunderbird
     >>> entries = parse_entries(browser.contents)
@@ -264,19 +267,20 @@ type of content as the latest bugs feed for a product.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs in Hoary']
     >>> browser.url
     'http://feeds.launchpad.test/ubuntu/hoary/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2006-10-16:/bugs/ubuntu/hoary
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/ubuntu/hoary/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/ubuntu/hoary/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -304,19 +308,20 @@ type of content as the latest bugs feed for a product.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs in 1.0']
     >>> browser.url
     'http://feeds.launchpad.test/firefox/1.0/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2005-06-06:/bugs/firefox/1.0
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/firefox/1.0/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/firefox/1.0/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -342,19 +347,20 @@ This feed gets the latest bugs for a person.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs for Foo Bar']
     >>> browser.url
     'http://feeds.launchpad.test/~name16/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2005-06-06:/bugs/~name16
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/~name16/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/~name16/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -417,17 +423,18 @@ some results.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs for Simple Team']
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2007-02-21:/bugs/~simple-team
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/~simple-team/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/~simple-team/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -445,19 +452,20 @@ This feed gets the latest bugs reported against any target.
     >>> validate_feed(browser.contents,
     ...               browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Launchpad bugs']
     >>> browser.url
     'http://feeds.launchpad.test/bugs/latest-bugs.atom'
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> print(extract_text(soup.find('id')))
     tag:launchpad.net,2008:/bugs
 
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/bugs/latest-bugs.atom" />
+    <link href="http://feeds.launchpad.test/bugs/latest-bugs.atom" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -508,10 +516,11 @@ The bug search feed can be tested after setting is_bug_search_feed_active
 to True.
 
     >>> browser.open(url)
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bugs from custom search']
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> feed_id = extract_text(soup.find('id'))
     >>> print(feed_id)
     tag:launchpad.net,2008:/+bugs.atom?field.scope.target=&amp;field.scope=all&amp;field.searchtext=&amp;search=Search+Bug+Reports
@@ -523,7 +532,7 @@ to True.
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/bugs/+bugs.atom?field.scope.target=&amp;field.scope=all&amp;field.searchtext=&amp;search=Search+Bug+Reports" />
+    <link href="http://feeds.launchpad.test/bugs/+bugs.atom?field.scope.target=&amp;field.scope=all&amp;field.searchtext=&amp;search=Search+Bug+Reports" rel="self"/>
 
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -554,7 +563,7 @@ This feed shows the status of a single bug.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Bug 1']
     >>> entries = parse_entries(browser.contents)
     >>> print(len(entries))
@@ -565,7 +574,7 @@ This feed shows the status of a single bug.
     >>> self_links = parse_links(browser.contents, 'self')
     >>> for link in self_links:
     ...     print(link)
-    <link rel="self" href="http://feeds.launchpad.test/bugs/1/bug.atom" />
+    <link href="http://feeds.launchpad.test/bugs/1/bug.atom" rel="self"/>
 
 == Feeds Configuration Options ==
 
diff --git a/lib/lp/bugs/stories/feeds/xx-bug-html.txt b/lib/lp/bugs/stories/feeds/xx-bug-html.txt
index fff1f4f..fa02653 100644
--- a/lib/lp/bugs/stories/feeds/xx-bug-html.txt
+++ b/lib/lp/bugs/stories/feeds/xx-bug-html.txt
@@ -5,15 +5,16 @@ The content of an HTML feed is very similar to an Atom feed, but is formatted
 as HTML instead of Atom.
 
     >>> from lp.services.beautifulsoup import (
-    ...     BeautifulSoup,
-    ...     SoupStrainer,
+    ...     BeautifulSoup4 as BeautifulSoup,
+    ...     SoupStrainer4 as SoupStrainer,
     ...     )
 
 Define a helper function for parsing the entries:
 
     >>> def parse_entries(contents):
-    ...     entries = [tag for tag in BeautifulSoup(browser.contents,
-    ...                   parseOnlyThese=SoupStrainer('tr'))]
+    ...     entries = [
+    ...         tag for tag in BeautifulSoup(
+    ...             browser.contents, parse_only=SoupStrainer('tr'))]
     ...     return entries
 
 And two for printing the results:
diff --git a/lib/lp/code/stories/feeds/xx-branch-atom.txt b/lib/lp/code/stories/feeds/xx-branch-atom.txt
index db90f8f..74db8e1 100644
--- a/lib/lp/code/stories/feeds/xx-branch-atom.txt
+++ b/lib/lp/code/stories/feeds/xx-branch-atom.txt
@@ -1,10 +1,12 @@
 = Atom Feeds For Branches =
 
 Atom feeds produce XML not HTML.  Therefore we must parse the output as XML
-using BeautifulStoneSoup instead of BeautifulSoup or the helper functions.
+by asking BeautifulSoup to use lxml.
 
-    >>> from BeautifulSoup import BeautifulStoneSoup as BSS
-    >>> from BeautifulSoup import SoupStrainer
+    >>> from lp.services.beautifulsoup import (
+    ...     BeautifulSoup4 as BeautifulSoup,
+    ...     SoupStrainer4 as SoupStrainer,
+    ...     )
     >>> from lp.services.feeds.tests.helper import (
     ...     parse_ids, parse_links, validate_feed)
 
@@ -49,7 +51,7 @@ which will include an entry for each branch.
     ...         browser.contents, browser.headers['content-type'], browser.url)
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Branches for Mike Murphy']
     >>> def print_parse_ids(browser):
     ...     for id in parse_ids(browser.contents):
@@ -71,14 +73,15 @@ Ensure the self link is correct and there is only one.
     ...     for link in parse_links(browser.contents, rel="self"):
     ...         print(link)
     >>> print_parse_links(anon_browser)
-    <link rel="self" href="http://feeds.launchpad.test/~mike/branches.atom" />
+    <link href="http://feeds.launchpad.test/~mike/branches.atom" rel="self"/>
 
 The <update> field for the feed will be the most recent value for the
 updated field in all of the entries.
 
     >>> strainer = SoupStrainer('updated')
-    >>> updated_dates = [extract_text(tag) for tag in BSS(anon_browser.contents,
-    ...                  parseOnlyThese=strainer)]
+    >>> updated_dates = [
+    ...     extract_text(tag) for tag in BeautifulSoup(
+    ...         anon_browser.contents, 'xml', parse_only=strainer)]
     >>> feed_updated = updated_dates[0]
     >>> entry_dates = sorted(updated_dates[1:], reverse=True)
     >>> assert feed_updated == entry_dates[0], (
@@ -90,7 +93,7 @@ still be hidden:
     >>> anon_browser.open('http://feeds.launchpad.test/~name12/branches.atom')
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Branches for Sample Person']
     >>> 'foo@localhost' in anon_browser.contents
     False
@@ -125,7 +128,7 @@ branches listed, just an id for the feed.
     >>> browser.open('http://feeds.launchpad.test/~landscape-developers/branches.atom')
     >>> validate_browser_feed(browser)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Branches for Landscape Developers']
     >>> print_parse_ids(browser)
     <id>tag:launchpad.net,2006-07-11:/code/~landscape-developers</id>
@@ -139,7 +142,7 @@ which will include an entry for each branch.
     >>> anon_browser.open('http://feeds.launchpad.test/fooix/branches.atom')
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Branches for Fooix']
     >>> print_parse_ids(anon_browser)
     <id>tag:launchpad.net,...:/code/fooix</id>
@@ -148,14 +151,15 @@ which will include an entry for each branch.
     <id>tag:launchpad.net,2007-12-01:/code/~mike/fooix/first</id>
 
     >>> print_parse_links(anon_browser)
-    <link rel="self" href="http://feeds.launchpad.test/fooix/branches.atom" />
+    <link href="http://feeds.launchpad.test/fooix/branches.atom" rel="self"/>
 
 The <update> field for the feed will be the most recent value for the
 updated field in all of the entries.
 
     >>> strainer = SoupStrainer('updated')
-    >>> updated_dates = [extract_text(tag) for tag in BSS(anon_browser.contents,
-    ...                  parseOnlyThese=strainer)]
+    >>> updated_dates = [
+    ...     extract_text(tag) for tag in BeautifulSoup(
+    ...         anon_browser.contents, 'xml', parse_only=strainer)]
     >>> feed_updated = updated_dates[0]
     >>> entry_dates = sorted(updated_dates[1:], reverse=True)
     >>> assert feed_updated == entry_dates[0], (
@@ -170,7 +174,7 @@ branches which will include an entry for each branch.
     >>> anon_browser.open('http://feeds.launchpad.test/oh-man/branches.atom')
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Branches for Oh Man']
     >>> print_parse_ids(anon_browser)
     <id>tag:launchpad.net,...:/code/oh-man</id>
@@ -182,14 +186,15 @@ branches which will include an entry for each branch.
     <id>tag:launchpad.net,2007-12-01:/code/~mike/fooix/first</id>
 
     >>> print_parse_links(anon_browser)
-    <link rel="self" href="http://feeds.launchpad.test/oh-man/branches.atom" />
+    <link href="http://feeds.launchpad.test/oh-man/branches.atom" rel="self"/>
 
 The <update> field for the feed will be the most recent value for the
 updated field in all of the entries.
 
     >>> strainer = SoupStrainer('updated')
-    >>> updated_dates = [extract_text(tag) for tag in BSS(anon_browser.contents,
-    ...                  parseOnlyThese=strainer)]
+    >>> updated_dates = [
+    ...     extract_text(tag) for tag in BeautifulSoup(
+    ...         anon_browser.contents, 'xml', parse_only=strainer)]
     >>> feed_updated = updated_dates[0]
     >>> entry_dates = sorted(updated_dates[1:], reverse=True)
     >>> assert feed_updated == entry_dates[0], (
@@ -206,7 +211,7 @@ different entry.
     >>> validate_feed(browser.contents,
     ...              browser.headers['content-type'], browser.url)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Latest Revisions for Branch lp://dev/~mark/firefox/release--0.9.1']
     >>> print(browser.url)
     http://feeds.launchpad.test/~mark/firefox/release--0.9.1/branch.atom
@@ -214,17 +219,19 @@ different entry.
 The first <id> in a feed identifies the feed.  Each entry then has its
 own <id>, which in the case of a single branch feed will be identical.
 
-    >>> soup = BSS(browser.contents, parseOnlyThese=SoupStrainer('id'))
+    >>> soup = BeautifulSoup(
+    ...     browser.contents, 'xml', parse_only=SoupStrainer('id'))
     >>> ids = parse_ids(browser.contents)
     >>> for id_ in ids:
     ...     print(id_)
     <id>tag:launchpad.net,2006-10-16:/code/~mark/firefox/release--0.9.1</id>
     <id>tag:launchpad.net,2005-03-09:/code/~mark/firefox/release--0.9.1/revision/1</id>
     >>> print_parse_links(browser)
-    <link rel="self" href="http://feeds.launchpad.test/~mark/firefox/release--0.9.1/branch.atom" />
+    <link href="http://feeds.launchpad.test/~mark/firefox/release--0.9.1/branch.atom" rel="self"/>
     >>> strainer = SoupStrainer('updated')
-    >>> updated_dates = [extract_text(tag) for tag in BSS(browser.contents,
-    ...                  parseOnlyThese=strainer)]
+    >>> updated_dates = [
+    ...     extract_text(tag) for tag in BeautifulSoup(
+    ...         browser.contents, 'xml', parse_only=strainer)]
 
 The update date for the entire feed (updated_dates[0]) must be equal
 to the update_date of the first entry in the feed (updated_dates[1]).
diff --git a/lib/lp/code/stories/feeds/xx-revision-atom.txt b/lib/lp/code/stories/feeds/xx-revision-atom.txt
index 0ef5eae..3638b09 100644
--- a/lib/lp/code/stories/feeds/xx-revision-atom.txt
+++ b/lib/lp/code/stories/feeds/xx-revision-atom.txt
@@ -1,9 +1,9 @@
 = Atom Feeds For Revisions =
 
 Atom feeds produce XML not HTML.  Therefore we must parse the output as XML
-using BeautifulStoneSoup instead of BeautifulSoup or the helper functions.
+by asking BeautifulSoup to use lxml.
 
-    >>> from BeautifulSoup import BeautifulStoneSoup as BSS
+    >>> from lp.services.beautifulsoup import BeautifulSoup4 as BeautifulSoup
     >>> from lp.services.feeds.tests.helper import (
     ...     parse_ids, parse_links, validate_feed)
 
@@ -75,7 +75,7 @@ that have been committed by that person (or attributed to that person).
     ...         browser.contents, browser.headers['content-type'], browser.url)
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Latest Revisions by Mike Murphy']
     >>> def print_parse_ids(browser):
     ...     for id in parse_ids(browser.contents):
@@ -96,7 +96,7 @@ Ensure the self link is correct and there is only one.
     ...     for link in parse_links(browser.contents, rel="self"):
     ...         print(link)
     >>> print_parse_links(anon_browser)
-    <link rel="self" href="http://feeds.launchpad.test/~mike/revisions.atom" />
+    <link href="http://feeds.launchpad.test/~mike/revisions.atom" rel="self"/>
 
 If we look at the feed for a team, we get revisions created by any member
 of that team.
@@ -104,7 +104,7 @@ of that team.
     >>> browser.open('http://feeds.launchpad.test/~m-team/revisions.atom')
     >>> validate_browser_feed(browser)
     No Errors
-    >>> BSS(browser.contents).title.contents
+    >>> BeautifulSoup(browser.contents, 'xml').title.contents
     [u'Latest Revisions by members of The M Team']
     >>> print_parse_ids(browser)
     <id>tag:launchpad.net,...:/code/~m-team</id>
@@ -122,7 +122,7 @@ that have been committed on branches for the product.
     >>> anon_browser.open('http://feeds.launchpad.test/fooix/revisions.atom')
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Latest Revisions for Fooix']
 
 Ignore the date associated with the id of 'fooix' as this is the date created
@@ -136,7 +136,7 @@ for the product, which will be different each time the test is run.
 Ensure the self link points to the feed location and there is only one.
 
     >>> print_parse_links(anon_browser)
-    <link rel="self" href="http://feeds.launchpad.test/fooix/revisions.atom" />
+    <link href="http://feeds.launchpad.test/fooix/revisions.atom" rel="self"/>
 
 
 == Feed for a project group's revisions ==
@@ -147,7 +147,7 @@ branch for any product that is associated with the project group.
     >>> anon_browser.open('http://feeds.launchpad.test/fubar/revisions.atom')
     >>> validate_browser_feed(anon_browser)
     No Errors
-    >>> BSS(anon_browser.contents).title.contents
+    >>> BeautifulSoup(anon_browser.contents, 'xml').title.contents
     [u'Latest Revisions for Fubar']
 
 Ignore the date associated with the id of 'fubar' as this is the date created
@@ -163,4 +163,4 @@ of the project group, which will be different each time the test is run.
 Ensure the self link points to the feed location and there is only one.
 
     >>> print_parse_links(anon_browser)
-    <link rel="self" href="http://feeds.launchpad.test/fubar/revisions.atom" />
+    <link href="http://feeds.launchpad.test/fubar/revisions.atom" rel="self"/>
diff --git a/lib/lp/registry/stories/announcements/xx-announcements.txt b/lib/lp/registry/stories/announcements/xx-announcements.txt
index f8e1045..addfc4a 100644
--- a/lib/lp/registry/stories/announcements/xx-announcements.txt
+++ b/lib/lp/registry/stories/announcements/xx-announcements.txt
@@ -7,8 +7,8 @@ dedicated batched page showing all announcements, and as an RSS/Atom
 news feed.
 
     >>> from lp.services.beautifulsoup import (
-    ...     BeautifulSoup,
-    ...     SoupStrainer,
+    ...     BeautifulSoup4 as BeautifulSoup,
+    ...     SoupStrainer4 as SoupStrainer,
     ...     )
     >>> from lp.services.feeds.tests.helper import (
     ...     parse_ids, parse_links, validate_feed)
@@ -643,7 +643,7 @@ domain.
     >>> links = parse_links(nopriv_browser.contents, rel='self')
     >>> for link in links:
     ...     print link
-    <link rel="self" href="http://feeds.launchpad.test/netapplet/announcements.atom" />
+    <link href="http://feeds.launchpad.test/netapplet/announcements.atom" rel="self"/>
 
     >>> for id_ in parse_ids(nopriv_browser.contents):
     ...     print extract_text(id_)
@@ -716,7 +716,7 @@ products.
     >>> links = parse_links(nopriv_browser.contents, rel='self')
     >>> for link in links:
     ...     print link
-    <link rel="self" href="http://feeds.launchpad.test/apache/announcements.atom" />
+    <link href="http://feeds.launchpad.test/apache/announcements.atom" rel="self"/>
 
 Finally, there is a feed for all announcements across all projects
 hosted in Launchpad:
@@ -755,18 +755,16 @@ let us use a DTD to define the html entities that standard xml is missing.
     No Errors
     >>> soup = BeautifulSoup(nopriv_browser.contents)
     >>> soup.find('feed').entry.title
-    <...>Ampersand=&quot;&amp;&quot; LessThan=&quot;&lt;&quot;
-    GreaterThan=&quot;&gt;&quot;</title>
-    >>> soup.find('feed').entry.content
+    <...>Ampersand="&amp;" LessThan="&lt;" GreaterThan="&gt;"</title>
+    >>> print(soup.find('feed').entry.content)
     <...
-    Ampersand=&amp;quot;&amp;amp;&amp;quot;&lt;br /&gt;
-    LessThan=&amp;quot;&amp;lt;&amp;quot;&lt;br /&gt;
-    GreaterThan=&amp;quot;&amp;gt;&amp;quot;&lt;br /&gt;
-    Newline=&amp;quot;&lt;br /&gt;
-    &amp;quot;&lt;br /&gt;
-    url=&amp;quot;&lt;a rel=&quot;nofollow&quot;
-    href=&quot;http://www.ubuntu.com&quot;&gt;http://&lt;wbr
-    /&gt;www.ubuntu.&lt;wbr /&gt;com&lt;/a&gt;&amp;quot;...
+    Ampersand="&amp;amp;"&lt;br/&gt;
+    LessThan="&amp;lt;"&lt;br/&gt;
+    GreaterThan="&amp;gt;"&lt;br/&gt;
+    Newline="&lt;br/&gt;
+    "&lt;br/&gt;
+    url="&lt;a href="http://www.ubuntu.com"
+    rel="nofollow"&gt;http://&lt;wbr/&gt;www.ubuntu.&lt;wbr/&gt;com&lt;/a&gt;"...
 
 
 Deletion
diff --git a/lib/lp/services/feeds/doc/feeds.txt b/lib/lp/services/feeds/doc/feeds.txt
index 35086d2..02315cb 100644
--- a/lib/lp/services/feeds/doc/feeds.txt
+++ b/lib/lp/services/feeds/doc/feeds.txt
@@ -157,7 +157,7 @@ we are testing xhtml encoding here in case we need it in the future.
     >>> xhtml = FeedTypedData("<b> and &nbsp; and &amp;</b><hr/>",
     ...               content_type="xhtml")
     >>> xhtml.content
-    u'<b> and \xa0 and &amp;</b><hr />'
+    u'<b> and \xa0 and &amp;</b><hr/>'
 
 
 == validate_feed() helper function ==
diff --git a/lib/lp/services/feeds/feed.py b/lib/lp/services/feeds/feed.py
index 9462062..143121c 100644
--- a/lib/lp/services/feeds/feed.py
+++ b/lib/lp/services/feeds/feed.py
@@ -27,7 +27,7 @@ from zope.component import getUtility
 from zope.datetime import rfc1123_date
 from zope.interface import implementer
 
-from lp.services.beautifulsoup import BeautifulSoup
+from lp.services.beautifulsoup import BeautifulSoup4 as BeautifulSoup
 from lp.services.config import config
 from lp.services.feeds.interfaces.feed import (
     IFeed,
@@ -302,9 +302,7 @@ class FeedTypedData:
         if self.content_type in ('text', 'html'):
             altered_content = html_escape(altered_content)
         elif self.content_type == 'xhtml':
-            soup = BeautifulSoup(
-                altered_content,
-                convertEntities=BeautifulSoup.HTML_ENTITIES)
+            soup = BeautifulSoup(altered_content)
             altered_content = unicode(soup)
         return altered_content
 
diff --git a/lib/lp/services/feeds/stories/xx-links.txt b/lib/lp/services/feeds/stories/xx-links.txt
index 83110fd..23e467b 100644
--- a/lib/lp/services/feeds/stories/xx-links.txt
+++ b/lib/lp/services/feeds/stories/xx-links.txt
@@ -11,13 +11,13 @@ launchpad.test to provide links to corresponding Atom feeds.
 The root launchpad.test url will have a link to the Atom feed which
 displays the most recent announcements for all the projects.
 
-    >>> from lp.services.beautifulsoup import BeautifulSoup
+    >>> from lp.services.beautifulsoup import BeautifulSoup4 as BeautifulSoup
     >>> browser.open('http://launchpad.test/')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/announcements.atom"
-        title="All Announcements" />]
+    [<link href="http://feeds.launchpad.test/announcements.atom"
+        rel="alternate" title="All Announcements"
+        type="application/atom+xml"/>]
 
 The http://launchpad.test/+announcements page also displays recent
 announcements for all the projects so it should have a link to the same
@@ -26,9 +26,9 @@ feed.
     >>> browser.open('http://launchpad.test/+announcements')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/announcements.atom"
-        title="All Announcements" />]
+    [<link href="http://feeds.launchpad.test/announcements.atom"
+        rel="alternate" title="All Announcements"
+        type="application/atom+xml"/>]
 
 == Single Bug Feed ==
 
@@ -38,9 +38,9 @@ atom feed for that one bug.
     >>> browser.open('http://bugs.launchpad.test/firefox/+bug/1')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/bugs/1/bug.atom"
-        title="Bug 1 Feed" />]
+    [<link href="http://feeds.launchpad.test/bugs/1/bug.atom"
+        rel="alternate" title="Bug 1 Feed"
+        type="application/atom+xml"/>]
 
 But if the bug is private, there should be no link.
 
@@ -80,15 +80,15 @@ branches.
     >>> browser.open('http://launchpad.test/~stevea')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-       href="http://feeds.launchpad.test/~stevea/latest-bugs.atom"
-       title="Latest Bugs for Steve Alexander" />,
-    <link rel="alternate" type="application/atom+xml"
-       href="http://feeds.launchpad.test/~stevea/branches.atom"
-       title="Latest Branches for Steve Alexander" />,
-    <link rel="alternate" type="application/atom+xml"
-       href="http://feeds.launchpad.test/~stevea/revisions.atom"
-       title="Latest Revisions by Steve Alexander" />]
+    [<link href="http://feeds.launchpad.test/~stevea/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for Steve Alexander"
+        type="application/atom+xml"/>,
+    <link href="http://feeds.launchpad.test/~stevea/branches.atom"
+        rel="alternate" title="Latest Branches for Steve Alexander"
+        type="application/atom+xml"/>,
+    <link href="http://feeds.launchpad.test/~stevea/revisions.atom"
+        rel="alternate" title="Latest Revisions by Steve Alexander"
+        type="application/atom+xml"/>]
 
 On the bugs subdomain, only a link to the bugs feed will be included,
 not the branches link.
@@ -96,9 +96,9 @@ not the branches link.
     >>> browser.open('http://bugs.launchpad.test/~stevea')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/~stevea/latest-bugs.atom"
-        title="Latest Bugs for Steve Alexander" />]
+    [<link href="http://feeds.launchpad.test/~stevea/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for Steve Alexander"
+        type="application/atom+xml"/>]
 
 
 == Latest Bugs, Branches, and Announcements for a Product ==
@@ -112,27 +112,27 @@ main product page.
     >>> browser.open('http://launchpad.test/jokosher')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/jokosher/announcements.atom"
-        title="Announcements for Jokosher" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/jokosher/latest-bugs.atom"
-        title="Latest Bugs for Jokosher" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/jokosher/branches.atom"
-        title="Latest Branches for Jokosher" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/jokosher/revisions.atom"
-        title="Latest Revisions for Jokosher" />]
+    [<link href="http://feeds.launchpad.test/jokosher/announcements.atom"
+        rel="alternate" title="Announcements for Jokosher"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/jokosher/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for Jokosher"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/jokosher/branches.atom"
+        rel="alternate" title="Latest Branches for Jokosher"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/jokosher/revisions.atom"
+        rel="alternate" title="Latest Revisions for Jokosher"
+        type="application/atom+xml"/>]
 
 Only bug feeds should be linked to on bugs.launchpad.test.
 
     >>> browser.open('http://bugs.launchpad.test/jokosher')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/jokosher/latest-bugs.atom"
-        title="Latest Bugs for Jokosher" />]
+    [<link href="http://feeds.launchpad.test/jokosher/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for Jokosher"
+        type="application/atom+xml"/>]
 
 
 == Escaping the title ==
@@ -160,18 +160,22 @@ it must have quotes and html escaped.
     >>> browser.open('http://launchpad.test/bad-displayname')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/bad-displayname/announcements.atom"
-        title='Announcements for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;' />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/bad-displayname/latest-bugs.atom"
-        title='Latest Bugs for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;' />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/bad-displayname/branches.atom"
-        title='Latest Branches for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;' />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/bad-displayname/revisions.atom"
-        title='Latest Revisions for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;' />]
+    [<link href="http://feeds.launchpad.test/bad-displayname/announcements.atom"
+        rel="alternate"
+        title='Announcements for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;'
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/bad-displayname/latest-bugs.atom"
+        rel="alternate"
+        title='Latest Bugs for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;'
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/bad-displayname/branches.atom"
+        rel="alternate"
+        title='Latest Branches for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;'
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/bad-displayname/revisions.atom"
+        rel="alternate"
+        title='Latest Revisions for Bad displayname"&gt;&lt;script&gt;alert("h4x0r")&lt;/script&gt;'
+        type="application/atom+xml"/>]
 
 == Latest Bugs for a ProjectGroup ==
 
@@ -184,27 +188,27 @@ on the main project group page.
     >>> browser.open('http://launchpad.test/gnome')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/gnome/announcements.atom"
-        title="Announcements for GNOME" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/gnome/latest-bugs.atom"
-        title="Latest Bugs for GNOME" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/gnome/branches.atom"
-        title="Latest Branches for GNOME" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/gnome/revisions.atom"
-        title="Latest Revisions for GNOME" />]
+    [<link href="http://feeds.launchpad.test/gnome/announcements.atom"
+        rel="alternate" title="Announcements for GNOME"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/gnome/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for GNOME"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/gnome/branches.atom"
+        rel="alternate" title="Latest Branches for GNOME"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/gnome/revisions.atom"
+        rel="alternate" title="Latest Revisions for GNOME"
+        type="application/atom+xml"/>]
 
 Only bug feeds should be linked to on bugs.launchpad.test.
 
     >>> browser.open('http://bugs.launchpad.test/gnome')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/gnome/latest-bugs.atom"
-        title="Latest Bugs for GNOME" />]
+    [<link href="http://feeds.launchpad.test/gnome/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for GNOME"
+        type="application/atom+xml"/>]
 
 The default view for a project group on bugs.launchpad.test is +bugs. The
 default bug listing matches the latest-bugs atom feed, but any search
@@ -231,21 +235,21 @@ An announcements feed link should also be shown on the main distro page.
     >>> browser.open('http://launchpad.test/ubuntu')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/ubuntu/announcements.atom"
-        title="Announcements for Ubuntu" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/ubuntu/latest-bugs.atom"
-        title="Latest Bugs for Ubuntu" />]
+    [<link href="http://feeds.launchpad.test/ubuntu/announcements.atom"
+        rel="alternate" title="Announcements for Ubuntu"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/ubuntu/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for Ubuntu"
+        type="application/atom+xml"/>]
 
 Only bug feeds should be linked to on bugs.launchpad.test.
 
     >>> browser.open('http://bugs.launchpad.test/ubuntu')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/ubuntu/latest-bugs.atom"
-        title="Latest Bugs for Ubuntu" />]
+    [<link href="http://feeds.launchpad.test/ubuntu/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for Ubuntu"
+        type="application/atom+xml"/>]
 
 
 == Latest Bugs for a Distroseries ==
@@ -256,9 +260,10 @@ show a link to the atom feed for that distroseries' latest bugs.
     >>> browser.open('http://bugs.launchpad.test/ubuntu/hoary')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
+    [<link
         href="http://feeds.launchpad.test/ubuntu/hoary/latest-bugs.atom"
-        title="Latest Bugs for Hoary" />]
+        rel="alternate" title="Latest Bugs for Hoary"
+        type="application/atom+xml"/>]
 
 
 == Latest Bugs for a Product Series ==
@@ -269,9 +274,9 @@ show a link to the atom feed for that product series' latest bugs.
     >>> browser.open('http://bugs.launchpad.test/firefox/1.0')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/firefox/1.0/latest-bugs.atom"
-        title="Latest Bugs for 1.0" />]
+    [<link href="http://feeds.launchpad.test/firefox/1.0/latest-bugs.atom"
+        rel="alternate" title="Latest Bugs for 1.0"
+        type="application/atom+xml"/>]
 
 
 == Latest Bugs for a Source Package ==
@@ -282,9 +287,10 @@ show a link to the atom feed for that source package's latest bugs.
     >>> browser.open('http://bugs.launchpad.test/ubuntu/+source/cnews')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
+    [<link
         href="http://feeds.launchpad.test/ubuntu/+source/cnews/latest-bugs.atom"
-        title="Latest Bugs for cnews in Ubuntu" />]
+        rel="alternate" title="Latest Bugs for cnews in Ubuntu"
+        type="application/atom+xml"/>]
 
 
 == Latest Branches for a ProjectGroup ==
@@ -295,12 +301,14 @@ to the atom feed for that project group's latest branches.
     >>> browser.open('http://code.launchpad.test/mozilla')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
+    [<link
         href="http://feeds.launchpad.test/mozilla/branches.atom"
-        title="Latest Branches for The Mozilla Project" />,
-     <link rel="alternate" type="application/atom+xml"
+        rel="alternate" title="Latest Branches for The Mozilla Project"
+        type="application/atom+xml"/>,
+     <link
         href="http://feeds.launchpad.test/mozilla/revisions.atom"
-        title="Latest Revisions for The Mozilla Project" />]
+        rel="alternate" title="Latest Revisions for The Mozilla Project"
+        type="application/atom+xml"/>]
 
 
 == Latest Branches for a Product ==
@@ -311,12 +319,13 @@ to the atom feed for that product's latest branches.
     >>> browser.open('http://code.launchpad.test/firefox')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/firefox/branches.atom"
-        title="Latest Branches for Mozilla Firefox" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/firefox/revisions.atom"
-        title="Latest Revisions for Mozilla Firefox" />]
+    [<link href="http://feeds.launchpad.test/firefox/branches.atom"
+        rel="alternate" title="Latest Branches for Mozilla Firefox"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/firefox/revisions.atom"
+        rel="alternate"
+        title="Latest Revisions for Mozilla Firefox"
+        type="application/atom+xml"/>]
 
 
 == Latest Branches for a Person ==
@@ -327,12 +336,12 @@ to the atom feed for that person's latest branches.
     >>> browser.open('http://code.launchpad.test/~mark')
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/~mark/branches.atom"
-        title="Latest Branches for Mark Shuttleworth" />,
-     <link rel="alternate" type="application/atom+xml"
-        href="http://feeds.launchpad.test/~mark/revisions.atom"
-        title="Latest Revisions by Mark Shuttleworth" />]
+    [<link href="http://feeds.launchpad.test/~mark/branches.atom"
+        rel="alternate" title="Latest Branches for Mark Shuttleworth"
+        type="application/atom+xml"/>,
+     <link href="http://feeds.launchpad.test/~mark/revisions.atom"
+        rel="alternate" title="Latest Revisions by Mark Shuttleworth"
+        type="application/atom+xml"/>]
 
 
 == Latest Revisions on a Branch ==
@@ -344,9 +353,11 @@ atom feed for that branch's revisions.
     >>> browser.open(url)
     >>> soup = BeautifulSoup(browser.contents)
     >>> soup.head.findAll('link', type='application/atom+xml')
-    [<link rel="alternate" type="application/atom+xml"
+    [<link
         href="http://feeds.launchpad.test/~mark/firefox/release--0.9.1/branch.atom"
-	title="Latest Revisions for Branch lp://dev/~mark/firefox/release--0.9.1" />]
+        rel="alternate"
+	title="Latest Revisions for Branch lp://dev/~mark/firefox/release--0.9.1"
+        type="application/atom+xml"/>]
 
 But if the branch is private, there should be no link.
 
diff --git a/lib/lp/services/feeds/stories/xx-security.txt b/lib/lp/services/feeds/stories/xx-security.txt
index ea7122d..2e441df 100644
--- a/lib/lp/services/feeds/stories/xx-security.txt
+++ b/lib/lp/services/feeds/stories/xx-security.txt
@@ -4,32 +4,32 @@ Feeds do not display private bugs
 Feeds never contain private bugs, as we are serving feeds over HTTP.
 First, set all the bugs to private.
 
-    >>> from zope.security.interfaces import Unauthorized
-    >>> from BeautifulSoup import BeautifulStoneSoup as BSS
-    >>> from lp.services.database.interfaces import IStore
     >>> import transaction
-    >>> from lp.bugs.model.bug import Bug
+    >>> from zope.security.interfaces import Unauthorized
     >>> from lp.app.enums import InformationType
+    >>> from lp.bugs.model.bug import Bug
+    >>> from lp.services.beautifulsoup import BeautifulSoup4 as BeautifulSoup
+    >>> from lp.services.database.interfaces import IStore
     >>> IStore(Bug).find(Bug).set(information_type=InformationType.USERDATA)
     >>> transaction.commit()
 
 There should be zero entries in these feeds, since all the bugs are private.
 
     >>> browser.open('http://feeds.launchpad.test/jokosher/latest-bugs.atom')
-    >>> BSS(browser.contents)('entry')
+    >>> BeautifulSoup(browser.contents, 'xml')('entry')
     []
 
     >>> browser.open('http://feeds.launchpad.test/mozilla/latest-bugs.atom')
-    >>> BSS(browser.contents)('entry')
+    >>> BeautifulSoup(browser.contents, 'xml')('entry')
     []
 
     >>> browser.open('http://feeds.launchpad.test/~name16/latest-bugs.atom')
-    >>> BSS(browser.contents)('entry')
+    >>> BeautifulSoup(browser.contents, 'xml')('entry')
     []
 
     >>> browser.open(
     ...     'http://feeds.launchpad.test/~simple-team/latest-bugs.atom')
-    >>> BSS(browser.contents)('entry')
+    >>> BeautifulSoup(browser.contents, 'xml')('entry')
     []
 
     >>> from lp.services.config import config
@@ -41,52 +41,52 @@ There should be zero entries in these feeds, since all the bugs are private.
     >>> browser.open('http://feeds.launchpad.test/bugs/+bugs.atom?'
     ...        'field.searchtext=&search=Search+Bug+Reports&'
     ...        'field.scope=all&field.scope.target=')
-    >>> BSS(browser.contents)('entry')
+    >>> BeautifulSoup(browser.contents, 'xml')('entry')
     []
 
 There should be just one <tr> elements for the table header in
 these HTML feeds, since all the bugs are private.
 
     >>> browser.open('http://feeds.launchpad.test/jokosher/latest-bugs.html')
-    >>> len(BSS(browser.contents)('tr'))
+    >>> len(BeautifulSoup(browser.contents, 'xml')('tr'))
     1
 
-    >>> print extract_text(BSS(browser.contents)('tr')[0])
+    >>> print extract_text(BeautifulSoup(browser.contents, 'xml')('tr')[0])
     Bugs in Jokosher
 
     >>> browser.open('http://feeds.launchpad.test/mozilla/latest-bugs.html')
-    >>> len(BSS(browser.contents)('tr'))
+    >>> len(BeautifulSoup(browser.contents, 'xml')('tr'))
     1
 
-    >>> print extract_text(BSS(browser.contents)('tr')[0])
+    >>> print extract_text(BeautifulSoup(browser.contents, 'xml')('tr')[0])
     Bugs in The Mozilla Project
 
     >>> browser.open('http://feeds.launchpad.test/~name16/latest-bugs.html')
-    >>> len(BSS(browser.contents)('tr'))
+    >>> len(BeautifulSoup(browser.contents, 'xml')('tr'))
     1
 
-    >>> print extract_text(BSS(browser.contents)('tr')[0])
+    >>> print extract_text(BeautifulSoup(browser.contents, 'xml')('tr')[0])
     Bugs for Foo Bar
 
     >>> browser.open(
     ...     'http://feeds.launchpad.test/~simple-team/latest-bugs.html')
-    >>> len(BSS(browser.contents)('tr'))
+    >>> len(BeautifulSoup(browser.contents, 'xml')('tr'))
     1
 
-    >>> print extract_text(BSS(browser.contents)('tr')[0])
+    >>> print extract_text(BeautifulSoup(browser.contents, 'xml')('tr')[0])
     Bugs for Simple Team
 
     >>> browser.open('http://feeds.launchpad.test/bugs/+bugs.html?'
     ...        'field.searchtext=&search=Search+Bug+Reports&'
     ...        'field.scope=all&field.scope.target=')
-    >>> len(BSS(browser.contents)('tr'))
+    >>> len(BeautifulSoup(browser.contents, 'xml')('tr'))
     1
 
     >>> try:
     ...      browser.open('http://feeds.launchpad.test/bugs/1/bug.html')
     ... except Unauthorized:
     ...     print "Shouldn't  raise Unauthorized exception"
-    >>> BSS(browser.contents)('entry')
+    >>> BeautifulSoup(browser.contents, 'xml')('entry')
     []
 
 Revert configuration change after tests are finished.
diff --git a/lib/lp/services/feeds/tests/helper.py b/lib/lp/services/feeds/tests/helper.py
index e0a96aa..826df83 100644
--- a/lib/lp/services/feeds/tests/helper.py
+++ b/lib/lp/services/feeds/tests/helper.py
@@ -31,9 +31,11 @@ from zope.interface import (
     implementer,
     Interface,
     )
-from BeautifulSoup import BeautifulStoneSoup as BSS
-from BeautifulSoup import SoupStrainer
 
+from lp.services.beautifulsoup import (
+    BeautifulSoup4 as BeautifulSoup,
+    SoupStrainer4 as SoupStrainer,
+    )
 from lp.services.webapp.publisher import LaunchpadView
 
 
@@ -62,25 +64,23 @@ class ThingFeedView(LaunchpadView):
 def parse_entries(contents):
     """Define a helper function for parsing feed entries."""
     strainer = SoupStrainer('entry')
-    entries = [tag for tag in BSS(contents,
-                                  parseOnlyThese=strainer)]
+    entries = [
+        tag for tag in BeautifulSoup(contents, 'xml', parse_only=strainer)]
     return entries
 
 
 def parse_links(contents, rel):
     """Define a helper function for parsing feed links."""
     strainer = SoupStrainer('link', rel=rel)
-    entries = [tag for tag in BSS(contents,
-                                  parseOnlyThese=strainer,
-                                  selfClosingTags=['link'])]
+    entries = [
+        tag for tag in BeautifulSoup(contents, 'xml', parse_only=strainer)]
     return entries
 
 
 def parse_ids(contents):
     """Define a helper function for parsing ids."""
     strainer = SoupStrainer('id')
-    ids = [tag for tag in BSS(contents,
-                              parseOnlyThese=strainer)]
+    ids = [tag for tag in BeautifulSoup(contents, 'xml', parse_only=strainer)]
     return ids