Merge lp:~mvo/software-center/merge-prev-purchases-lp969273 into lp:software-center

[Michael Vogt]

This branch changes the way the reinstall previous purchases is done. It merges them on update of the
update-software-center-update tool into the per-user xapian database instead of adding them as a seperate
in-memory database. This fixes the double entries in the DB and has the nice side-effect that items already
purchased will be displayed as already purchased instead of showing up with a price.

[lp:~mvo/software-center/merge-prev-purchases-lp969273]

3187. By Michael Vogt on 2012-09-14

tests/gtk3/test_purchase.py: update test to latest code

3188. By Michael Vogt on 2012-09-14

trivial pep8 fixes

[Michael Vogt (mvo)]

Careful review appreciated, should be good, but there is quite a bit of churn.

[Gary Lasker (gary-lasker)]

I looked this over carefully and tested reinstall previous purchases under various sequences of steps. I like the new query for a purchased item, it's simpler and it's very fast. All unit test changes look good and all changed tests pass for me.

Thanks, Michael!

review: Approve

=== modified file 'softwarecenter/backend/login.py'
--- softwarecenter/backend/login.py	2012-03-19 14:20:55 +0000
+++ softwarecenter/backend/login.py	2012-09-14 13:44:26 +0000
@@ -49,5 +49,11 @@
     def login(self, username=None, password=None):
         raise NotImplemented
 
+    def login_or_register(self):
+        raise NotImplemented
+
+    def find_credentials(self):
+        raise NotImplemented
+
     def cancel_login(self):
         self.emit("login-canceled")
 
=== modified file 'softwarecenter/backend/login_sso.py'
--- softwarecenter/backend/login_sso.py	2012-07-09 13:33:51 +0000
+++ softwarecenter/backend/login_sso.py	2012-09-14 13:44:26 +0000
@@ -31,6 +31,12 @@
 from softwarecenter.utils import utf8
 from login import LoginBackend
 
+from ubuntu_sso import (
+    DBUS_BUS_NAME,
+    DBUS_CREDENTIALS_IFACE,
+    DBUS_CREDENTIALS_PATH,
+    )
+
 # mostly for testing
 from fake_review_settings import FakeReviewSettings, network_delay
 
@@ -44,10 +50,15 @@
         self.appname = appname
         self.help_text = help_text
         self.bus = dbus.SessionBus()
-        self.proxy = self.bus.get_object(
-            'com.ubuntu.sso', '/com/ubuntu/sso/credentials')
+        obj = self.bus.get_object(bus_name=DBUS_BUS_NAME,
+                                  object_path=DBUS_CREDENTIALS_PATH,
+                                  follow_name_owner_changes=True)
+        self.proxy = dbus.Interface(object=obj,
+                               dbus_interface=DBUS_CREDENTIALS_IFACE)
         self.proxy.connect_to_signal("CredentialsFound",
                                      self._on_credentials_found)
+        self.proxy.connect_to_signal("CredentialsNotFound",
+                                     self._on_credentials_not_found)
         self.proxy.connect_to_signal("CredentialsError",
                                      self._on_credentials_error)
         self.proxy.connect_to_signal("AuthorizationDenied",
@@ -64,6 +75,11 @@
             p['window_id'] = self._window_id
         return p
 
+    def find_credentials(self):
+        LOG.debug("find_crendentials()")
+        self._credentials = None
+        self.proxy.find_credentials(self.appname, self._get_params())
+
     def login(self, username=None, password=None):
         LOG.debug("login()")
         self._credentials = None
@@ -74,6 +90,12 @@
         self._credentials = None
         self.proxy.register(self.appname, self._get_params())
 
+    def _on_credentials_not_found(self, app_name):
+        LOG.debug("_on_credentials_found for '%s'" % app_name)
+        if app_name != self.appname:
+            return
+        self.emit("login-failed")
+
     def _on_credentials_found(self, app_name, credentials):
         LOG.debug("_on_credentials_found for '%s'" % app_name)
         if app_name != self.appname:
 
=== modified file 'softwarecenter/backend/scagent.py'
--- softwarecenter/backend/scagent.py	2012-08-29 12:07:14 +0000
+++ softwarecenter/backend/scagent.py	2012-09-14 13:44:26 +0000
@@ -93,13 +93,14 @@
     def _on_query_available_data(self, spawner, piston_available):
         self.emit("available", piston_available)
 
-    def query_available_for_me(self):
+    def query_available_for_me(self, no_relogin=False):
         spawner = SpawnHelper()
         spawner.parent_xid = self.xid
         spawner.ignore_cache = self.ignore_cache
         spawner.connect("data-available", self._on_query_available_for_me_data)
         spawner.connect("error", lambda spawner, err: self.emit("error", err))
         spawner.needs_auth = True
+        spawner.no_relogin = no_relogin
         spawner.run_generic_piston_helper(
             "SoftwareCenterAgentAPI", "subscriptions_for_me",
             complete_only=True)
 
=== modified file 'softwarecenter/backend/spawn_helper.py'
--- softwarecenter/backend/spawn_helper.py	2012-05-16 15:52:07 +0000
+++ softwarecenter/backend/spawn_helper.py	2012-09-14 13:44:26 +0000
@@ -64,6 +64,7 @@
         self._child_watch = None
         self._cmd = None
         self.needs_auth = False
+        self.no_relogin = False
         self.ignore_cache = False
         self.parent_xid = None
 
@@ -76,6 +77,8 @@
             cmd.append("--needs-auth")
         if self.ignore_cache:
             cmd.append("--ignore-cache")
+        if self.no_relogin:
+            cmd.append("--no-relogin")
         if self.parent_xid:
             cmd.append("--parent-xid")
             cmd.append(str(self.parent_xid))
 
=== modified file 'softwarecenter/backend/ubuntusso.py'
--- softwarecenter/backend/ubuntusso.py	2012-08-30 08:57:29 +0000
+++ softwarecenter/backend/ubuntusso.py	2012-09-14 13:44:26 +0000
@@ -21,22 +21,44 @@
 
 
 from gi.repository import GObject
+from gettext import gettext as _
 
 import logging
 import os
 
+import piston_mini_client.auth
+import piston_mini_client.failhandlers
+
+
 import softwarecenter.paths
 
 # mostly for testing
 from fake_review_settings import FakeReviewSettings, network_delay
 from spawn_helper import SpawnHelper
 from softwarecenter.config import get_config
+from softwarecenter.backend.login_sso import get_sso_backend
+
+from softwarecenter.backend.piston.ubuntusso_pristine import (
+    UbuntuSsoAPI as PristineUbuntuSsoAPI,
+    )
+# patch default_service_root to the one we use
+from softwarecenter.enums import UBUNTU_SSO_SERVICE
+# *Don't* append /api/1.0, as it's already included in UBUNTU_SSO_SERVICE
+PristineUbuntuSsoAPI.default_service_root = UBUNTU_SSO_SERVICE
+
+from softwarecenter.enums import (SOFTWARE_CENTER_NAME_KEYRING,
+                                  SOFTWARE_CENTER_SSO_DESCRIPTION,
+                                  )
+from softwarecenter.utils import clear_token_from_ubuntu_sso_sync
 
 LOG = logging.getLogger(__name__)
 
 
-class UbuntuSSOAPI(GObject.GObject):
-    """ Ubuntu SSO interface using the oauth token from the keyring """
+class UbuntuSSO(GObject.GObject):
+    """ Ubuntu SSO interface using the oauth token from the keyring
+
+    The methods that work syncronous are suffixed with _sync()
+    """
 
     __gsignals__ = {
         "whoami": (GObject.SIGNAL_RUN_LAST,
@@ -49,8 +71,11 @@
                   ),
         }
 
-    def __init__(self):
+    def __init__(self, xid=0):
         GObject.GObject.__init__(self)
+        self.oauth = None
+        self.xid = xid
+        self.loop = GObject.MainLoop(GObject.main_context_default())
 
     def _on_whoami_data(self, spawner, piston_whoami):
         # once we have data, make sure to save it
@@ -71,11 +96,83 @@
         spawner.needs_auth = True
         spawner.run_generic_piston_helper("UbuntuSsoAPI", "whoami")
 
-
-class UbuntuSSOAPIFake(UbuntuSSOAPI):
+    def _login_successful(self, sso_backend, oauth_result):
+        LOG.debug("_login_successful")
+        self.oauth = oauth_result
+        self.loop.quit()
+
+    # sync calls
+    def verify_token_sync(self, token):
+        """ Verify that the token is valid
+
+            Note that this may raise httplib2 exceptions if the server
+            is not reachable
+        """
+        LOG.debug("verify_token")
+        auth = piston_mini_client.auth.OAuthAuthorizer(
+            token["token"], token["token_secret"],
+            token["consumer_key"], token["consumer_secret"])
+        api = PristineUbuntuSsoAPI(auth=auth)
+        try:
+            res = api.whoami()
+        except piston_mini_client.failhandlers.APIError as e:
+            LOG.exception("api.whoami failed with APIError: '%s'" % e)
+            return False
+        return len(res) > 0
+
+    def clear_token(self):
+        clear_token_from_ubuntu_sso_sync(SOFTWARE_CENTER_NAME_KEYRING)
+
+    def _get_sso_backend_and_connect(self):
+        sso = get_sso_backend(
+            self.xid,
+            SOFTWARE_CENTER_NAME_KEYRING,
+            _(SOFTWARE_CENTER_SSO_DESCRIPTION))
+        sso.connect("login-successful", self._login_successful)
+        sso.connect("login-failed", lambda s: self.loop.quit())
+        sso.connect("login-canceled", lambda s: self.loop.quit())
+        return sso
+
+    def find_oauth_token_sync(self):
+        self.oauth = None
+        sso = self. _get_sso_backend_and_connect()
+        sso.find_credentials()
+        self.loop.run()
+        return self.oauth
+
+    def get_oauth_token_sync(self):
+        self.oauth = None
+        sso = self. _get_sso_backend_and_connect()
+        sso.login_or_register()
+        self.loop.run()
+        return self.oauth
+
+    def get_oauth_token_and_verify_sync(self, no_relogin=False):
+        token = self.get_oauth_token_sync()
+        # check if the token is valid and reset it if it is not
+        if token:
+            # verify token will return false if there is a API error,
+            # but there maybe httplib2 errors if there is no network,
+            # so ignore them
+            try:
+                if not self.verify_token_sync(token):
+                    attempt_relogin = not no_relogin
+                    if attempt_relogin:
+                        self.clear_token()
+                        # re-trigger login once
+                        token = self.get_oauth_token_sync()
+                    else:
+                        return None
+            except Exception as e:
+                LOG.warn(
+                    "token could not be verified (network problem?): %s" % e)
+        return token
+
+
+class UbuntuSSOAPIFake(UbuntuSSO):
 
     def __init__(self):
-        UbuntuSSOAPI.__init__(self)
+        UbuntuSSO.__init__(self)
         self._fake_settings = FakeReviewSettings()
 
     @network_delay
@@ -110,7 +207,7 @@
         ubuntu_sso_class = UbuntuSSOAPIFake()
         LOG.warn('Using fake Ubuntu SSO API. Only meant for testing purposes')
     else:
-        ubuntu_sso_class = UbuntuSSOAPI()
+        ubuntu_sso_class = UbuntuSSO()
     return ubuntu_sso_class
 
 
@@ -133,6 +230,7 @@
     password = sys.stdin.readline().strip()
     sso.login(user, password)
 
+
 # interactive test code
 if __name__ == "__main__":
     def _whoami(sso, result):
@@ -145,7 +243,7 @@
 
     def _dbus_maybe_login_successful(ssologin, oauth_result):
         print "got token, verify it now"
-        sso = UbuntuSSOAPI()
+        sso = UbuntuSSO()
         sso.connect("whoami", _whoami)
         sso.connect("error", _error)
         sso.whoami()
@@ -154,7 +252,6 @@
     logging.basicConfig(level=logging.DEBUG)
     softwarecenter.paths.datadir = "./data"
 
-    from login_sso import get_sso_backend
     backend = get_sso_backend("", "appname", "help_text")
     backend.connect("login-successful", _dbus_maybe_login_successful)
     backend.login_or_register()
 
=== modified file 'softwarecenter/db/database.py'
--- softwarecenter/db/database.py	2012-09-12 12:41:05 +0000
+++ softwarecenter/db/database.py	2012-09-14 13:44:26 +0000
@@ -43,6 +43,18 @@
 LOG = logging.getLogger(__name__)
 
 
+def get_reinstall_previous_purchases_query():
+    """Return a query to get applications purchased
+
+    :return: a xapian query to get all the apps that are purchaed
+    """
+    # this query will give us all documents that have a purchase date != ""
+    query = xapian.Query(xapian.Query.OP_VALUE_GE,
+                         XapianValues.PURCHASED_DATE,
+                         "1")
+    return query
+
+
 def parse_axi_values_file(filename="/var/lib/apt-xapian-index/values"):
     """ parse the apt-xapian-index "values" file and provide the
     information in the self._axi_values dict
 
=== modified file 'softwarecenter/db/update.py'
--- softwarecenter/db/update.py	2012-09-10 09:40:48 +0000
+++ softwarecenter/db/update.py	2012-09-14 13:44:26 +0000
@@ -30,9 +30,13 @@
 from gi.repository import GObject
 from piston_mini_client import PistonResponseObject
 
+from softwarecenter.backend.scagent import SoftwareCenterAgent
+from softwarecenter.backend.ubuntusso import UbuntuSSO
 from softwarecenter.distro import get_distro
 from softwarecenter.utils import utf8
 
+from gettext import gettext as _
+
 # py3 compat
 try:
     from configparser import RawConfigParser, NoOptionError
@@ -49,7 +53,6 @@
     import pickle
 
 
-from gettext import gettext as _
 from glob import glob
 from urlparse import urlparse
 
@@ -59,7 +62,6 @@
     AppInfoFields,
     AVAILABLE_FOR_PURCHASE_MAGIC_CHANNEL_NAME,
     DB_SCHEMA_VERSION,
-    PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME,
     XapianValues,
 )
 from softwarecenter.db.database import parse_axi_values_file
@@ -672,8 +674,6 @@
         #          gets confused about (appname, pkgname) duplication
         self.sca_application.name = utf8(_("%s (already purchased)")) % utf8(
             self.sca_application.name)
-        self.sca_application.channel = (
-            PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME)
         for attr_name in ('license_key', 'license_key_path'):
             attr = getattr(self.sca_subscription, attr_name, self.NOT_DEFINED)
             if attr is self.NOT_DEFINED:
@@ -1007,44 +1007,6 @@
     return True
 
 
-def add_from_purchased_but_needs_reinstall_data(
-    purchased_but_may_need_reinstall_list, db, cache):
-    """Add application that have been purchased but may require a reinstall
-
-    This adds a inmemory database to the main db with the special
-    PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME channel prefix
-
-    :return: a xapian query to get all the apps that need reinstall
-    """
-    # magic
-    db_purchased = xapian.inmemory_open()
-    # go over the items we have
-    for item in purchased_but_may_need_reinstall_list:
-        # FIXME: what to do with duplicated entries? we will end
-        #        up with two xapian.Document, one for the for-pay
-        #        and one for the availalbe one from s-c-agent
-        #try:
-        #    db.get_xapian_document(item.name,
-        #                           item.package_name)
-        #except IndexError:
-        #    # item is not in the xapian db
-        #    pass
-        #else:
-        #    # ignore items we already have in the db, ignore
-        #    continue
-        # index the item
-        try:
-            parser = SCAPurchasedApplicationParser(item)
-            parser.index_app_info(db_purchased, cache)
-        except:
-            LOG.exception("error processing: %r", item)
-    # add new in memory db to the main db
-    db.add_database(db_purchased)
-    # return a query
-    query = xapian.Query("AH" + PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME)
-    return query
-
-
 def update_from_software_center_agent(db, cache, ignore_cache=False,
                                       include_sca_qa=False):
     """Update the index based on the software-center-agent data."""
@@ -1056,31 +1018,60 @@
         sca.good_data = True
         loop.quit()
 
+    def _available_for_me_cb(sca, available_for_me):
+        LOG.debug("update_from_software_center_agent: available_for_me: %r",
+                  available_for_me)
+        sca.available_for_me = available_for_me
+        loop.quit()
+
     def _error_cb(sca, error):
         LOG.warn("update_from_software_center_agent: error: %r", error)
-        sca.available = []
         sca.good_data = False
         loop.quit()
 
-    # use the anonymous interface to s-c-agent, scales much better and is
-    # much cache friendlier
-    from softwarecenter.backend.scagent import SoftwareCenterAgent
-    # FIXME: honor ignore_etag here somehow with the new piston based API
+    context = GObject.main_context_default()
+    loop = GObject.MainLoop(context)
+
     sca = SoftwareCenterAgent(ignore_cache)
     sca.connect("available", _available_cb)
+    sca.connect("available-for-me", _available_for_me_cb)
     sca.connect("error", _error_cb)
-    sca.available = None
+    sca.available = []
+    sca.available_for_me = []
+
+    # query what is available for me first
+    available_for_me_pkgnames = set()
+    # this will ensure we do not trigger a login dialog
+    helper = UbuntuSSO()
+    token = helper.find_oauth_token_sync()
+    if token:
+        sca.query_available_for_me(no_relogin=True)
+        loop.run()
+        for item in sca.available_for_me:
+            try:
+                parser = SCAPurchasedApplicationParser(item)
+                parser.index_app_info(db, cache)
+                available_for_me_pkgnames.add(item.application["package_name"])
+            except:
+                LOG.exception("error processing: %r", item)
+
+    # ... now query all that is available
     if include_sca_qa:
         sca.query_available_qa()
     else:
         sca.query_available()
+
     # create event loop and run it until data is available
     # (the _available_cb and _error_cb will quit it)
-    context = GObject.main_context_default()
-    loop = GObject.MainLoop(context)
     loop.run()
+
     # process data
     for entry in sca.available:
+
+        # do not add stuff here thats already purchased to avoid duplication
+        if entry.package_name in available_for_me_pkgnames:
+            continue
+
         # process events
         while context.pending():
             context.iteration()
 
=== modified file 'softwarecenter/enums.py'
--- softwarecenter/enums.py	2012-09-11 00:02:18 +0000
+++ softwarecenter/enums.py	2012-09-14 13:44:26 +0000
@@ -208,7 +208,6 @@
 
 
 # fake channels
-PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME = "for-pay-needs-reinstall"
 AVAILABLE_FOR_PURCHASE_MAGIC_CHANNEL_NAME = "available-for-pay"
 
 
 
=== modified file 'softwarecenter/ui/gtk3/app.py'
--- softwarecenter/ui/gtk3/app.py	2012-09-11 13:31:09 +0000
+++ softwarecenter/ui/gtk3/app.py	2012-09-14 13:44:26 +0000
@@ -47,6 +47,8 @@
 # make pyflakes shut up
 softwarecenter.netstatus.NETWORK_STATE
 
+from softwarecenter.backend.ubuntusso import UbuntuSSO
+
 # db imports
 from softwarecenter.db.application import Application
 from softwarecenter.db import (
@@ -84,7 +86,10 @@
     init_sc_css_provider,
 )
 from softwarecenter.version import VERSION
-from softwarecenter.db.database import StoreDatabase
+from softwarecenter.db.database import (
+    StoreDatabase,
+    get_reinstall_previous_purchases_query,
+    )
 try:
     from aptd_gtk3 import InstallBackendUI
     InstallBackendUI  # pyflakes
@@ -336,7 +341,7 @@
         # for use when viewing previous purchases
         self.scagent = None
         self.sso = None
-        self.available_for_me_query = None
+        self.available_for_me_query = get_reinstall_previous_purchases_query()
 
         Gtk.Window.set_default_icon_name("softwarecenter")
 
@@ -739,25 +744,9 @@
         channel_manager.feed_in_private_sources_list_entries(
             private_archives)
 
-    def _on_sso_login(self, sso, oauth_result):
-        self._sso_login_successful = True
-        # appmanager needs to know about the oauth token for the reinstall
-        # previous purchases add_license_key call
-        self.app_manager.oauth_token = oauth_result
-        self.scagent.query_available_for_me()
-
     def _on_style_updated(self, widget, init_css_callback, *args):
         init_css_callback(widget, *args)
 
-    def _available_for_me_result(self, scagent, result_list):
-        #print "available_for_me_result", result_list
-        from softwarecenter.db.update import (
-            add_from_purchased_but_needs_reinstall_data)
-        available = add_from_purchased_but_needs_reinstall_data(result_list,
-            self.db, self.cache)
-        self.available_for_me_query = available
-        self.available_pane.on_previous_purchases_activated(available)
-
     def get_icon_filename(self, iconname, iconsize):
         iconinfo = self.icons.lookup_icon(iconname, iconsize, 0)
         if not iconinfo:
@@ -838,30 +827,53 @@
         d = LoginDialog(self.glaunchpad, self.datadir, parent=self.window_main)
         d.login()
 
-    def _create_dbus_sso(self):
-        # see bug #773214 for the rationale, do not translate the appname
-        #appname = _("Ubuntu Software Center")
-        appname = SOFTWARE_CENTER_NAME_KEYRING
-        help_text = _("To reinstall previous purchases, sign in to the "
-            "Ubuntu Single Sign-On account you used to pay for them.")
-        #window = self.window_main.get_window()
-        #xid = self.get_window().xid
-        xid = 0
-        self.sso = get_sso_backend(xid,
-                                   appname,
-                                   help_text)
-        self.sso.connect("login-successful", self._on_sso_login)
-
-    def _login_via_dbus_sso(self):
-        self._create_dbus_sso()
-        self.sso.login()
-
-    def _create_scagent_if_needed(self):
-        if not self.scagent:
-            from softwarecenter.backend.scagent import SoftwareCenterAgent
-            self.scagent = SoftwareCenterAgent()
-            self.scagent.connect("available-for-me",
-                                 self._available_for_me_result)
+    def _on_reinstall_purchased_login(self, sso, oauth_result):
+        self._sso_login_successful = True
+        # appmanager needs to know about the oauth token for the reinstall
+        # previous purchases add_license_key call
+        self.app_manager.oauth_token = oauth_result
+
+        # the software-center-agent will ensure the previous-purchases
+        # get merged in
+        self._run_software_center_agent()
+
+        # show spinner as this may take some time, the spinner will
+        # automatically go away when a DB refreshes
+        self.available_pane.show_appview_spinner()
+
+        # show previous purchased
+        self.available_pane.on_previous_purchases_activated(
+            self.available_for_me_query)
+
+    def on_menuitem_reinstall_purchases_activate(self, menuitem):
+        self.view_manager.set_active_view(ViewPages.AVAILABLE)
+        self.view_manager.search_entry.clear_with_no_signal()
+        # its ok to use the sync version here to get the token, this is
+        # very quick
+        helper = UbuntuSSO()
+        token = helper.find_oauth_token_sync()
+        if token:
+            # trigger a software-center-agent run to ensure the merged in
+            # purchases are fresh
+            self._run_software_center_agent()
+            # we already have the list of available items, so just show it
+            # (no need for spinner here)
+            self.available_pane.on_previous_purchases_activated(
+                    self.available_for_me_query)
+        else:
+            # see bug #773214 for the rationale, do not translate the appname
+            #appname = _("Ubuntu Software Center")
+            appname = SOFTWARE_CENTER_NAME_KEYRING
+            help_text = _(
+                "To reinstall previous purchases, sign in to the "
+                "Ubuntu Single Sign-On account you used to pay for them.")
+            #window = self.window_main.get_window()
+            #xid = self.get_window().xid
+            xid = 0
+            self.sso = get_sso_backend(xid, appname, help_text)
+            self.sso.connect(
+                "login-successful", self._on_reinstall_purchased_login)
+            self.sso.login()
 
     def on_menuitem_recommendations_activate(self, menu_item):
         rec_panel = self.available_pane.cat_view.recommended_for_you_panel
@@ -875,21 +887,6 @@
             if res == Gtk.ResponseType.YES:
                 rec_panel.opt_in_to_recommendations_service()
 
-    def on_menuitem_reinstall_purchases_activate(self, menuitem):
-        self.view_manager.set_active_view(ViewPages.AVAILABLE)
-        self.view_manager.search_entry.clear_with_no_signal()
-        if self.available_for_me_query:
-            # we already have the list of available items, so just show it
-            # (no need for spinner here)
-            self.available_pane.on_previous_purchases_activated(
-                    self.available_for_me_query)
-        else:
-            # show spinner as this may take some time
-            self.available_pane.show_appview_spinner()
-            # fetch the list of available items and show it
-            self._create_scagent_if_needed()
-            self._login_via_dbus_sso()
-
     def on_menuitem_deauthorize_computer_activate(self, menuitem):
 
         # FIXME: need Ubuntu SSO username here
 
=== modified file 'tests/gtk3/test_purchase.py'
--- tests/gtk3/test_purchase.py	2012-08-21 08:46:26 +0000
+++ tests/gtk3/test_purchase.py	2012-09-14 13:44:26 +0000
@@ -71,7 +71,13 @@
         do_events_with_sleep()
         self.assertTrue(signal_mock.called)
 
-    def test_reinstall_previous_purchase_display(self):
+
+class PreviousPurchasesTestCase(unittest.TestCase):
+
+    @patch("softwarecenter.backend.ubuntusso.UbuntuSSO"
+           ".find_oauth_token_sync")
+    def test_reinstall_previous_purchase_display(self, mock_find_token):
+        mock_find_token.return_value = { 'not': 'important' }
         mock_options = get_mock_options()
         xapiandb = "/var/cache/software-center/"
         app = SoftwareCenterAppGtk3(
@@ -79,22 +85,19 @@
         self.addCleanup(app.destroy)
         # real app opens cache async
         app.cache.open()
-        # no real sso
-        with patch.object(app, '_login_via_dbus_sso',
-                          lambda: app._available_for_me_result(None, [])):
-            # show it
-            app.window_main.show_all()
-            app.available_pane.init_view()
+        # .. and now pretend we clicked on the menu item
+        app.window_main.show_all()
+        app.available_pane.init_view()
+        do_events_with_sleep()
+        app.on_menuitem_reinstall_purchases_activate(None)
+        # it can take a bit until the sso client is ready
+        for i in range(10):
+            if (app.available_pane.get_current_page() ==
+                AvailablePane.Pages.LIST):
+                break
             do_events_with_sleep()
-            app.on_menuitem_reinstall_purchases_activate(None)
-            # it can take a bit until the sso client is ready
-            for i in range(100):
-                if (app.available_pane.get_current_page() ==
-                    AvailablePane.Pages.LIST):
-                    break
-                do_events_with_sleep()
-            self.assertEqual(app.available_pane.get_current_page(),
-                             AvailablePane.Pages.LIST)
+        self.assertEqual(app.available_pane.get_current_page(),
+                         AvailablePane.Pages.LIST)
 
 
 if __name__ == "__main__":
 
=== modified file 'tests/test_database.py'
--- tests/test_database.py	2012-09-14 07:31:06 +0000
+++ tests/test_database.py	2012-09-14 13:44:26 +0000
@@ -166,7 +166,11 @@
         self.assertTrue(res)
         self.assertEqual(db.get_doccount(), 1)
 
-    def test_build_from_software_center_agent(self):
+    @patch("softwarecenter.backend.ubuntusso.UbuntuSSO"
+           ".find_oauth_token_sync")
+    def test_build_from_software_center_agent(self, mock_find_oauth):
+        # pretend we have no token
+        mock_find_oauth.return_value = None
         db = xapian.inmemory_open()
         cache = apt.Cache()
         # monkey patch distro to ensure we get data
@@ -365,7 +369,11 @@
             doc.get_value(value_time) >= last_time
             last_time = doc.get_value(value_time)
 
-    def test_for_purchase_apps_date_published(self):
+    @patch("softwarecenter.backend.ubuntusso.UbuntuSSO"
+           ".find_oauth_token_sync")
+    def test_for_purchase_apps_date_published(self, mock_find_oauth):
+        # pretend we have no token
+        mock_find_oauth.return_value = None
         #os.environ["SOFTWARE_CENTER_DEBUG_HTTP"] = "1"
         #os.environ["SOFTWARE_CENTER_AGENT_HOST"] = "http://sc.staging.ubuntu.com/"
         # staging does not have a valid cert
 
=== modified file 'tests/test_reinstall_purchased.py'
--- tests/test_reinstall_purchased.py	2012-06-25 17:40:17 +0000
+++ tests/test_reinstall_purchased.py	2012-09-14 13:44:26 +0000
@@ -1,30 +1,29 @@
-import apt_pkg
-import apt
 import json
 import platform
-import os
 import unittest
 import xapian
 
+from gi.repository import GObject
+
 from mock import patch
 from piston_mini_client import PistonResponseObject
 from tests.utils import (
-    DATA_DIR,
+    get_test_pkg_info,
     setup_test_env,
+    ObjectWithSignals,
 )
 setup_test_env()
 
 from softwarecenter.enums import (
     AppInfoFields,
     AVAILABLE_FOR_PURCHASE_MAGIC_CHANNEL_NAME,
-    PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME,
     XapianValues,
 )
-from softwarecenter.db.database import StoreDatabase
+from softwarecenter.db.database import get_reinstall_previous_purchases_query
 from softwarecenter.db.update import (
-    add_from_purchased_but_needs_reinstall_data,
     SCAPurchasedApplicationParser,
     SCAApplicationParser,
+    update_from_software_center_agent,
 )
 
 # Example taken from running:
@@ -98,62 +97,45 @@
         "description": "Play DVD-Videos\\r\\n\\r\\nFluendo DVD Player is a software application specially designed to\\r\\nreproduce DVD on Linux/Unix platforms, which provides end users with\\r\\nhigh quality standards.\\r\\n\\r\\nThe following features are provided:\\r\\n* Full DVD Playback\\r\\n* DVD Menu support\\r\\n* Fullscreen support\\r\\n* Dolby Digital pass-through\\r\\n* Dolby Digital 5.1 output and stereo downmixing support\\r\\n* Resume from last position support\\r\\n* Subtitle support\\r\\n* Audio selection support\\r\\n* Multiple Angles support\\r\\n* Support for encrypted discs\\r\\n* Multiregion, works in all regions\\r\\n* Multiple video deinterlacing algorithms",
         "website": null,
         "version": "1.2.1"
-    }
+    },
+    {
+    "website": "",
+    "package_name": "photobomb",
+    "video_embedded_html_urls": [ ],
+    "demo": null,
+    "keywords": "photos, pictures, editing, gwibber, twitter, facebook, drawing",
+    "video_urls": [ ],
+    "screenshot_url": "http://software-center.ubuntu.com/site_media/screenshots/2011/08/Screenshot-45.png",
+    "id": 83,
+    "archive_id": "commercial-ppa-uploaders/photobomb",
+    "support_url": "http://launchpad.net/photobomb",
+    "icon_url": "http://software-center.ubuntu.com/site_media/icons/2011/08/logo_64.png",
+    "binary_filesize": null,
+    "version": "",
+    "company_name": "",
+    "department": [
+        "Graphics"
+    ],
+    "tos_url": "",
+    "channel": "For Purchase",
+    "status": "Published",
+    "signing_key_id": "1024R/75254D99",
+    "description": "Easy and Social Image Editor\\nPhotobomb give you easy access to images in your social networking feeds, pictures on your computer and peripherals, and pictures on the web, and let's you draw, write, crop, combine, and generally have a blast mashing 'em all up. Then you can save off your photobomb, or tweet your creation right back to your social network.",
+    "price": "2.99",
+    "debtags": [ ],
+    "date_published": "2011-12-05 18:43:20.794802",
+    "categories": "Graphics",
+    "name": "Photobomb",
+    "license": "GNU GPL v3",
+    "screenshot_urls": [
+        "http://software-center.ubuntu.com/site_media/screenshots/2011/08/Screenshot-45.png"
+    ],
+    "archive_root": "https://private-ppa.launchpad.net/"
+  }
 ]
 """
 
 
-class TestPurchased(unittest.TestCase):
-    """ tests the store database """
-
-    def _make_available_for_me_list(self):
-        my_subscriptions = json.loads(SUBSCRIPTIONS_FOR_ME_JSON)
-        return list(
-            PistonResponseObject.from_dict(subs) for subs in my_subscriptions)
-
-    def setUp(self):
-        # use fixture apt data
-        apt_pkg.config.set("APT::Architecture", "i386")
-        apt_pkg.config.set("Dir::State::status",
-            os.path.join(DATA_DIR, "appdetails", "var", "lib", "dpkg", "status"))
-        # create mocks
-        self.available_to_me = self._make_available_for_me_list()
-        self.cache = apt.Cache()
-
-    def test_reinstall_purchased_mock(self):
-        # test if the mocks are ok
-        self.assertEqual(len(self.available_to_me), 1)
-        self.assertEqual(
-            self.available_to_me[0].application['package_name'], "photobomb")
-
-    def test_reinstall_purchased_xapian(self):
-        db = StoreDatabase("/var/cache/software-center/xapian", self.cache)
-        db.open(use_axi=False)
-        # now create purchased debs xapian index (in memory because
-        # we store the repository passwords in here)
-        old_db_len = len(db)
-        query = add_from_purchased_but_needs_reinstall_data(
-            self.available_to_me, db, self.cache)
-        # ensure we have a new item (the available for reinstall one)
-        self.assertEqual(len(db), old_db_len+1)
-        # query
-        enquire = xapian.Enquire(db.xapiandb)
-        enquire.set_query(query)
-        matches = enquire.get_mset(0, len(db))
-        self.assertEqual(len(matches), 1)
-        distroseries = platform.dist()[2]
-        for m in matches:
-            doc = db.xapiandb.get_document(m.docid)
-            self.assertEqual(doc.get_value(XapianValues.PKGNAME), "photobomb")
-            self.assertEqual(
-                doc.get_value(XapianValues.ARCHIVE_SIGNING_KEY_ID),
-                "1024R/75254D99")
-            self.assertEqual(doc.get_value(XapianValues.ARCHIVE_DEB_LINE),
-                "deb https://username:random3atoken@"
-                 "private-ppa.launchpad.net/commercial-ppa-uploaders"
-                 "/photobomb/ubuntu %s main" % distroseries)
-
-
 class SCAApplicationParserTestCase(unittest.TestCase):
 
     def _make_application_parser(self, piston_application=None):
@@ -313,12 +295,11 @@
         for key in (AppInfoFields.LICENSE_KEY, AppInfoFields.LICENSE_KEY_PATH):
             self.assertIsNone(parser.get_value(key))
 
-    def test_magic_channel(self):
+    def test_purchase_date(self):
         parser = self._make_application_parser()
-
         self.assertEqual(
-            PURCHASED_NEEDS_REINSTALL_MAGIC_CHANNEL_NAME,
-            parser.get_value(AppInfoFields.CHANNEL))
+            "2011-09-16 06:37:52",
+            parser.get_value(AppInfoFields.PURCHASED_DATE))
 
     def test_will_handle_supported_distros_when_available(self):
         # When the fix for bug 917109 reaches production, we will be
@@ -367,5 +348,73 @@
             SCAPurchasedApplicationParser.update_debline(orig_debline))
 
 
+class TestAvailableForMeMerging(unittest.TestCase):
+
+    def setUp(self):
+        self.available_for_me = self._make_available_for_me_list()
+        self.available = self._make_available_list()
+
+    def _make_available_for_me_list(self):
+        my_subscriptions = json.loads(SUBSCRIPTIONS_FOR_ME_JSON)
+        return list(
+            PistonResponseObject.from_dict(subs) for subs in my_subscriptions)
+
+    def _make_available_list(self):
+        available_apps = json.loads(AVAILABLE_APPS_JSON)
+        return list(
+            PistonResponseObject.from_dict(subs) for subs in available_apps)
+
+    def _make_fake_scagent(self, available_data, available_for_me_data):
+        sca = ObjectWithSignals()
+        sca.query_available = lambda **kwargs: GObject.timeout_add(
+                100, lambda: sca.emit('available', sca, available_data))
+        sca.query_available_for_me = lambda **kwargs: GObject.timeout_add(
+            100, lambda: sca.emit('available-for-me', 
+                                  sca, available_for_me_data))
+        return sca
+
+    def test_reinstall_purchased_mock(self):
+        # test if the mocks are ok
+        self.assertEqual(len(self.available_for_me), 1)
+        self.assertEqual(
+            self.available_for_me[0].application['package_name'], "photobomb")
+
+    @patch("softwarecenter.db.update.SoftwareCenterAgent")
+    @patch("softwarecenter.db.update.UbuntuSSO")
+    def test_reinstall_purchased_xapian(self, mock_helper, mock_agent):
+        small_available =  [ self.available[0] ]
+        mock_agent.return_value = self._make_fake_scagent(
+            small_available, self.available_for_me)
+
+        db = xapian.inmemory_open()
+        cache = get_test_pkg_info()
+
+        # now create purchased debs xapian index (in memory because
+        # we store the repository passwords in here)
+        old_db_len = db.get_doccount()
+        update_from_software_center_agent(db, cache)
+        # ensure we have the new item
+        self.assertEqual(db.get_doccount(), old_db_len+2)
+        # query
+        query = get_reinstall_previous_purchases_query()
+        enquire = xapian.Enquire(db)
+        enquire.set_query(query)
+        matches = enquire.get_mset(0, db.get_doccount())
+        self.assertEqual(len(matches), 1)
+        distroseries = platform.dist()[2]
+        for m in matches:
+            doc = db.get_document(m.docid)
+            self.assertEqual(doc.get_value(XapianValues.PKGNAME), "photobomb")
+            self.assertEqual(
+                doc.get_value(XapianValues.ARCHIVE_SIGNING_KEY_ID),
+                "1024R/75254D99")
+            self.assertEqual(doc.get_value(XapianValues.ARCHIVE_DEB_LINE),
+                "deb https://username:random3atoken@"
+                 "private-ppa.launchpad.net/commercial-ppa-uploaders"
+                 "/photobomb/ubuntu %s main" % distroseries)
+
+
 if __name__ == "__main__":
+    import logging
+    logging.basicConfig(level=logging.DEBUG)
     unittest.main()
 
=== modified file 'tests/test_ubuntu_sso_api.py'
--- tests/test_ubuntu_sso_api.py	2012-05-30 18:39:55 +0000
+++ tests/test_ubuntu_sso_api.py	2012-09-14 13:44:26 +0000
@@ -6,7 +6,7 @@
 )
 setup_test_env()
 from softwarecenter.backend.ubuntusso import (UbuntuSSOAPIFake,
-                                              UbuntuSSOAPI,
+                                              UbuntuSSO,
                                               get_ubuntu_sso_backend,
                                               )
 
@@ -15,7 +15,7 @@
 
     def test_fake_and_real_provide_similar_methods(self):
         """ test if the real and fake sso provide the same functions """
-        sso_real = UbuntuSSOAPI
+        sso_real = UbuntuSSO
         sso_fake = UbuntuSSOAPIFake
         # ensure that both fake and real implement the same methods
         self.assertEqual(
@@ -25,7 +25,7 @@
     def test_get_ubuntu_backend(self):
         # test that we get the real one
         self.assertEqual(type(get_ubuntu_sso_backend()),
-                         UbuntuSSOAPI)
+                         UbuntuSSO)
         # test that we get the fake one
         os.environ["SOFTWARE_CENTER_FAKE_REVIEW_API"] = "1"
         self.assertEqual(type(get_ubuntu_sso_backend()),
 
=== modified file 'utils/piston-helpers/piston_generic_helper.py'
--- utils/piston-helpers/piston_generic_helper.py	2012-08-15 08:36:50 +0000
+++ utils/piston-helpers/piston_generic_helper.py	2012-09-14 13:44:26 +0000
@@ -25,7 +25,6 @@
 import pickle
 import sys
 
-from gi.repository import GObject
 
 # useful for debugging
 if "SOFTWARE_CENTER_DEBUG_HTTP" in os.environ:
@@ -45,13 +44,8 @@
 
 import softwarecenter.paths
 from softwarecenter.paths import SOFTWARE_CENTER_CACHE_DIR
-from softwarecenter.backend.login_sso import get_sso_backend
 
-from softwarecenter.enums import (SOFTWARE_CENTER_NAME_KEYRING,
-                                  SOFTWARE_CENTER_SSO_DESCRIPTION,
-                                  )
-                                  
-from softwarecenter.utils import clear_token_from_ubuntu_sso_sync
+from softwarecenter.backend.ubuntusso import UbuntuSSO
 
 # the piston import
 from softwarecenter.backend.piston.ubuntusso_pristine import UbuntuSsoAPI
@@ -60,90 +54,23 @@
 from softwarecenter.backend.piston.sreclient_pristine import (
     SoftwareCenterRecommenderAPI)
 
+
+from softwarecenter.enums import RECOMMENDER_HOST
+SoftwareCenterRecommenderAPI.default_service_root =  \
+    RECOMMENDER_HOST + "/api/1.0"
+
+
 # patch default_service_root to the one we use
 from softwarecenter.enums import UBUNTU_SSO_SERVICE
 # *Don't* append /api/1.0, as it's already included in UBUNTU_SSO_SERVICE
 UbuntuSsoAPI.default_service_root = UBUNTU_SSO_SERVICE
 
-from softwarecenter.enums import RECOMMENDER_HOST
-SoftwareCenterRecommenderAPI.default_service_root =  \
-    RECOMMENDER_HOST + "/api/1.0"
-
 
 RatingsAndReviewsAPI # pyflakes
 UbuntuSsoAPI # pyflakes
 SoftwareCenterAgentAPI # pyflakes
 SoftwareCenterRecommenderAPI # pyflakes
 
-from gettext import gettext as _
-
-
-# helper that is only used to verify that the token is ok
-# and trigger cleanup if not
-class SSOLoginHelper(object):
-
-    def __init__(self, xid=0):
-        self.oauth = None
-        self.xid = xid
-        self.loop = GObject.MainLoop(GObject.main_context_default())
-    
-    def _login_successful(self, sso_backend, oauth_result):
-        LOG.debug("_login_successful")
-        self.oauth = oauth_result
-        # FIXME: actually verify the token against ubuntu SSO
-        self.loop.quit()
-
-    def verify_token_sync(self, token):
-        """ Verify that the token is valid
-        
-            Note that this may raise httplib2 exceptions if the server
-            is not reachable
-        """
-        LOG.debug("verify_token")
-        auth = piston_mini_client.auth.OAuthAuthorizer(
-            token["token"], token["token_secret"],
-            token["consumer_key"], token["consumer_secret"])
-        api = UbuntuSsoAPI(auth=auth)
-        try:
-            res = api.whoami()
-        except piston_mini_client.failhandlers.APIError as e:
-            LOG.exception("api.whoami failed with APIError: '%s'" % e)
-            return False
-        return len(res) > 0
-
-    def clear_token(self):
-        clear_token_from_ubuntu_sso_sync(SOFTWARE_CENTER_NAME_KEYRING)
-
-    def get_oauth_token_sync(self):
-        self.oauth = None
-        sso = get_sso_backend(
-            self.xid, 
-            SOFTWARE_CENTER_NAME_KEYRING,
-            _(SOFTWARE_CENTER_SSO_DESCRIPTION))
-        sso.connect("login-successful", self._login_successful)
-        sso.connect("login-failed", lambda s: self.loop.quit())
-        sso.connect("login-canceled", lambda s: self.loop.quit())
-        sso.login_or_register()
-        self.loop.run()
-        return self.oauth
-
-    def get_oauth_token_and_verify_sync(self):
-        token = self.get_oauth_token_sync()
-        # check if the token is valid and reset it if it is not
-        if token:
-            # verify token will return false if there is a API error, 
-            # but there maybe httplib2 errors if there is no network,
-            # so ignore them
-            try:
-                if not self.verify_token_sync(token):
-                    self.clear_token()
-                    # re-trigger login once
-                    token = self.get_oauth_token_sync()
-            except Exception as e:
-                LOG.warn(
-                    "token could not be verified (network problem?): %s" % e)
-        return token
-
 
 LOG = logging.getLogger(__name__)
 
@@ -164,6 +91,8 @@
                         help="force disable offline mode")
     parser.add_argument("--needs-auth", default=False, action="store_true",
                         help="need oauth credentials")
+    parser.add_argument("--no-relogin", default=False, action="store_true",
+                        help="do not attempt relogin if token is invalid")
     parser.add_argument("--output", default="pickle",
                         help="output result as [pickle|json|text]")
     parser.add_argument("--parent-xid", default=0,
@@ -191,8 +120,9 @@
     softwarecenter.paths.datadir = args.datadir
 
     if args.needs_auth:
-        helper = SSOLoginHelper(args.parent_xid)
-        token = helper.get_oauth_token_and_verify_sync()
+        helper = UbuntuSSO(args.parent_xid)
+        token = helper.get_oauth_token_and_verify_sync(
+            no_relogin=args.no_relogin)
         # if we don't have a token, error here
         if not token:
             # it may happen that the parent is closed already so the pipe
 
=== modified file 'utils/update-software-center-agent'
--- utils/update-software-center-agent	2012-03-16 11:09:25 +0000
+++ utils/update-software-center-agent	2012-09-14 13:44:26 +0000
@@ -96,6 +96,10 @@
                      "a write lock on %s" % pathname)
         sys.exit(1)
 
+    # ensure permissions are 0700 as this may contain repo passwords from 
+    # the reinstall-previous-purchase repos
+    os.chmod(pathname, 0o700)
+
     # the following requires a http connection, so we do it in a 
     # seperate database
     include_sca_qa = "SOFTWARE_CENTER_AGENT_INCLUDE_QA" in os.environ