Merge lp:~stolowski/unity-scope-click/clear-cache-on-lang-change into lp:unity-scope-click

[Paweł Stołowski]

Clear network cache if language was changed

Clear network cache if language was changed

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:446
http://jenkins.qa.ubuntu.com/job/unity-scope-click-ci/738/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-scope-click-wily-amd64-ci/142/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-scope-click-wily-armhf-ci/138/console

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/unity-scope-click-ci/738/rebuild

review: Needs Fixing (continuous-integration)

[lp:~stolowski/unity-scope-click/clear-cache-on-lang-change]

447. By Paweł Stołowski on 2016-05-06

Added fixme comment

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:447
http://jenkins.qa.ubuntu.com/job/unity-scope-click-ci/739/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-scope-click-wily-amd64-ci/143/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-scope-click-wily-armhf-ci/139/console

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/unity-scope-click-ci/739/rebuild

review: Needs Fixing (continuous-integration)

[lp:~stolowski/unity-scope-click/clear-cache-on-lang-change]

448. By Paweł Stołowski on 2016-05-06

Moved to network access manager

449. By Paweł Stołowski on 2016-05-06

Merged network-timestamps

Unmerged revisions

=== modified file 'CMakeLists.txt'
--- CMakeLists.txt	2016-02-24 14:55:08 +0000
+++ CMakeLists.txt	2016-05-06 14:29:14 +0000
@@ -27,6 +27,9 @@
 pkg_check_modules(UNITY_SCOPES REQUIRED libunity-scopes>=0.6.7 libunity-api>=0.1.3)
 add_definitions(${UNITY_SCOPES_CFLAGS} ${UNITY_SCOPES_CFLAGS_OTHER})
 
+# Uncomment to enable network timings messages via stderr
+# add_definitions(-DNETWORK_TIMINGS)
+
 pkg_check_modules(UBUNTUONE REQUIRED ubuntuoneauth-2.0>=15.10)
 add_definitions(${UBUNTUONE_CFLAGS} ${UBUNTUONE_CFLAGS_OTHER})
 

=== modified file 'libclickscope/click/index.cpp'
--- libclickscope/click/index.cpp	2016-03-23 21:04:12 +0000
+++ libclickscope/click/index.cpp	2016-05-06 14:29:14 +0000
@@ -148,13 +148,14 @@
 }
 
 click::web::Cancellable Index::search (const std::string& query, const std::string& department,
-                                       std::function<void(click::Packages search_results, click::Packages recommendations)> callback)
+                                       std::function<void(click::Packages search_results, click::Packages recommendations)> callback,
+                                       bool force_cache)
 {
     click::web::CallParams params;
     const std::string built_query(build_index_query(query, department));
     params.add(click::QUERY_ARGNAME, built_query.c_str());
     QSharedPointer<click::web::Response> response(client->call(
-        get_base_url() + click::SEARCH_PATH, "GET", true, build_headers(), "", params));
+        get_base_url() + click::SEARCH_PATH, "GET", true, build_headers(), "", params, force_cache));
 
     QObject::connect(response.data(), &click::web::Response::finished, [=](QString reply) {
             std::pair<Packages, Packages> package_lists;
@@ -172,16 +173,17 @@
     return click::web::Cancellable(response);
 }
 
-click::web::Cancellable Index::bootstrap(std::function<void(const click::DepartmentList&, const click::HighlightList&, Error, int)> callback)
+click::web::Cancellable Index::bootstrap(std::function<void(const click::DepartmentList&, const click::HighlightList&, Error, int)> callback, bool force_cache)
 {
-    return departments(get_base_url() + click::BOOTSTRAP_PATH, callback);
+    return departments(get_base_url() + click::BOOTSTRAP_PATH, callback, force_cache);
 }
 
-click::web::Cancellable Index::departments(const std::string& department_href, std::function<void(const DepartmentList&, const HighlightList&, Error, int)> callback)
+click::web::Cancellable Index::departments(const std::string& department_href, std::function<void(const DepartmentList&, const HighlightList&, Error, int)>
+        callback, bool force_cache)
 {
     click::web::CallParams params;
     QSharedPointer<click::web::Response> response(client->call(
-        department_href, "GET", true, build_headers(), "", params));
+        department_href, "GET", true, build_headers(), "", params, force_cache));
 
     QObject::connect(response.data(), &click::web::Response::finished, [=](QString reply) {
             qDebug() << "departments request finished";
@@ -213,10 +215,12 @@
     return click::web::Cancellable(response);
 }
 
-click::web::Cancellable Index::get_details (const std::string& package_name, std::function<void(PackageDetails, click::Index::Error)> callback)
+click::web::Cancellable Index::get_details (const std::string& package_name, std::function<void(PackageDetails, click::Index::Error)> callback, bool force_cache)
 {
     QSharedPointer<click::web::Response> response = client->call
-        (get_base_url() + click::DETAILS_PATH + package_name);
+        (get_base_url() + click::DETAILS_PATH + package_name,
+         click::web::CallParams(),
+         force_cache);
     qDebug() << "getting details for" << package_name.c_str();
 
     QObject::connect(response.data(), &click::web::Response::finished, [=](const QByteArray reply) {

=== modified file 'libclickscope/click/index.h'
--- libclickscope/click/index.h	2015-12-01 15:39:41 +0000
+++ libclickscope/click/index.h	2016-05-06 14:29:14 +0000
@@ -79,10 +79,12 @@
     Index(const QSharedPointer<click::web::Client>& client,
           const QSharedPointer<Configuration> configuration=QSharedPointer<Configuration>(new Configuration()));
     virtual std::pair<Packages, Packages> package_lists_from_json(const std::string& json);
-    virtual click::web::Cancellable search (const std::string& query, const std::string& department, std::function<void(Packages, Packages)> callback);
-    virtual click::web::Cancellable get_details(const std::string& package_name, std::function<void(PackageDetails, Error)> callback);
-    virtual click::web::Cancellable bootstrap(std::function<void(const DepartmentList&, const HighlightList&, Error, int)> callback);
-    virtual click::web::Cancellable departments(const std::string& department_href, std::function<void(const DepartmentList&, const HighlightList&, Error, int)> callback);
+    virtual click::web::Cancellable search (const std::string& query, const std::string& department, std::function<void(Packages, Packages)> callback, bool
+            force_cache = false);
+    virtual click::web::Cancellable get_details(const std::string& package_name, std::function<void(PackageDetails, Error)> callback, bool force_cache = false);
+    virtual click::web::Cancellable bootstrap(std::function<void(const DepartmentList&, const HighlightList&, Error, int)> callback, bool force_cache = false);
+    virtual click::web::Cancellable departments(const std::string& department_href, std::function<void(const DepartmentList&, const HighlightList&, Error, int)>
+            callback, bool force_cache = false);
     virtual ~Index();
 
     virtual std::string get_suggested_currency() const;

=== modified file 'libclickscope/click/network_access_manager.cpp'
--- libclickscope/click/network_access_manager.cpp	2014-08-11 11:18:49 +0000
+++ libclickscope/click/network_access_manager.cpp	2016-05-06 14:29:14 +0000
@@ -30,8 +30,14 @@
 #include "network_access_manager.h"
 #include <QNetworkDiskCache>
 #include <QStandardPaths>
+#include <click/configuration.h>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <QDateTime>
+#include <time.h>
 
-click::network::Reply::Reply(QNetworkReply* reply) : reply(reply)
+click::network::Reply::Reply(QNetworkReply* reply, int id) : reply(reply)
 {
     connect(this->reply.data(),
             &QNetworkReply::finished,
@@ -43,6 +49,25 @@
             static_cast<QNetworkReplyErrorSignal>(&QNetworkReply::error),
             this,
             &Reply::error);
+
+#ifdef NETWORK_TIMINGS
+    auto url = reply->url().toString().toStdString();
+    connect(this->reply.data(),
+            &QNetworkReply::finished,
+            [url, id, reply] () {
+                bool fromCache = reply->attribute(QNetworkRequest::SourceIsFromCacheAttribute).toBool();
+                std::cerr << ">> " << id << " " << QDateTime::currentMSecsSinceEpoch() << " FINISHED: " << url << " cache:" << fromCache << std::endl;
+            });
+
+
+    connect(this->reply.data(),
+            static_cast<QNetworkReplyErrorSignal>(&QNetworkReply::error),
+            [url, id] () {
+                std::cerr << ">> " << id << QDateTime::currentMSecsSinceEpoch() << " ERROR: " << url << std::endl;
+            });
+#else
+    Q_UNUSED(id);
+#endif
 }
 
 click::network::Reply::~Reply()
@@ -90,6 +115,7 @@
 
 namespace
 {
+
 QNetworkAccessManager& networkAccessManagerInstance()
 {
     static QNetworkAccessManager nam;
@@ -98,27 +124,78 @@
         QNetworkDiskCache* cache = new QNetworkDiskCache(&nam);
         cache->setCacheDirectory(QString("%1/unity-scope-click/network").arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation)));
         nam.setCache(cache);
+
+        // FIXME: workaround for https://bugreports.qt.io/browse/QTBUG-14750 (no support for Vary header),
+        // should be removed once Qt network cache implements it.
+        if (click::network::AccessManager::languageChanged()) {
+            qDebug() << "Language change detected, clearing network cache";
+            nam.cache()->clear();
+        }
     }
     return nam;
 }
 }
 
+bool click::network::AccessManager::languageChanged()
+{
+    const QString langFilePath = QString("%1/unity-scope-click/language").arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation));
+    std::string lastLanguage;
+    {
+        std::ifstream lastLanguageFile(langFilePath.toStdString());
+        if (lastLanguageFile) {
+            std::stringstream ss;
+            ss << lastLanguageFile.rdbuf();
+            lastLanguage = ss.str();
+        }
+    }
+
+    auto const langs = Configuration().get_accept_languages();
+    if (lastLanguage != langs) {
+        std::ofstream lastLanguageFile(langFilePath.toStdString(), std::ios::out|std::ios::trunc);
+        lastLanguageFile << langs;
+        if (!lastLanguageFile) {
+            qWarning() << "Failed to write language file";
+        }
+        return true;
+    }
+    return false;
+}
+
+static int request_id = static_cast<int>(time(nullptr)); // this ensures request ids will be unique even if scope is restarted
+
 QSharedPointer<click::network::Reply> click::network::AccessManager::get(QNetworkRequest& request)
 {
-    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().get(request)));
+    int id = ++request_id;
+#ifdef NETWORK_TIMINGS
+    std::cerr << ">> " << id << " " << QDateTime::currentMSecsSinceEpoch() << " GET: " << request.url().toString().toStdString() << std::endl;
+#endif
+    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().get(request), id));
 }
 
 QSharedPointer<click::network::Reply> click::network::AccessManager::head(QNetworkRequest& request)
 {
-    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().head(request)));
+    int id = ++request_id;
+#ifdef NETWORK_TIMINGS
+    std::cerr << ">> " << id << " " << QDateTime::currentMSecsSinceEpoch() << " HEAD: " << request.url().toString().toStdString() << std::endl;
+#endif
+    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().head(request), id));
 }
 
 QSharedPointer<click::network::Reply> click::network::AccessManager::post(QNetworkRequest& request, QByteArray& data)
 {
-    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().post(request, data)));
+    int id = ++request_id;
+#ifdef NETWORK_TIMINGS
+    std::cerr << ">> " << id << " " << QDateTime::currentMSecsSinceEpoch() << " POST: " << request.url().toString().toStdString() << std::endl;
+#endif
+    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().post(request, data), id));
 }
 
 QSharedPointer<click::network::Reply> click::network::AccessManager::sendCustomRequest(QNetworkRequest& request, QByteArray& verb, QIODevice *data)
 {
-    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().sendCustomRequest(request, verb, data)));
+    int id = ++request_id;
+#ifdef NETWORK_TIMINGS
+    std::cerr << ">> " << id << " " << QDateTime::currentMSecsSinceEpoch() << " CUSTOM: " << request.url().toString().toStdString() << std::endl;
+#endif
+    return QSharedPointer<click::network::Reply>(new click::network::Reply(networkAccessManagerInstance().sendCustomRequest(request, verb, data), id));
 }
+

=== modified file 'libclickscope/click/network_access_manager.h'
--- libclickscope/click/network_access_manager.h	2014-05-20 19:33:41 +0000
+++ libclickscope/click/network_access_manager.h	2016-05-06 14:29:14 +0000
@@ -51,7 +51,7 @@
 
 public:
     // A Reply instance takes over ownership of the underlying QNetworkReply.
-    explicit Reply(QNetworkReply* reply);
+    explicit Reply(QNetworkReply* reply, int id = 0);
     Reply(const Reply&) = delete;
     virtual ~Reply();
 
@@ -93,6 +93,7 @@
     virtual QSharedPointer<Reply> head(QNetworkRequest& request);
     virtual QSharedPointer<Reply> post(QNetworkRequest& request, QByteArray& data);
     virtual QSharedPointer<Reply> sendCustomRequest(QNetworkRequest& request, QByteArray& verb, QIODevice *data = 0);
+    static bool languageChanged();
 };
 }
 }

=== modified file 'libclickscope/click/preview.cpp'
--- libclickscope/click/preview.cpp	2016-04-01 17:16:05 +0000
+++ libclickscope/click/preview.cpp	2016-05-06 14:29:14 +0000
@@ -215,7 +215,7 @@
             } else {
                 qWarning() << "unexpected action id " << QString::fromStdString(action_id)
                            << " given with download_url" << QString::fromStdString(download_url);
-                return new UninstalledPreview(result, client, depts, manager, ppackage);
+                return new UninstalledPreview(result, metadata, client, depts, manager, ppackage);
             }
         } else if (metadict.count(click::Preview::Actions::CANCEL_PURCHASE_UNINSTALLED) != 0) {
             return new CancelPurchasePreview(result, false);
@@ -224,18 +224,18 @@
         } else if (metadict.count(click::Preview::Actions::UNINSTALL_CLICK) != 0) {
             return new UninstallConfirmationPreview(result);
         } else if (metadict.count(click::Preview::Actions::CONFIRM_UNINSTALL) != 0) {
-            return new UninstallingPreview(result, client, manager, ppackage);
+            return new UninstallingPreview(result, metadata, client, manager, ppackage);
         } else if (metadict.count(click::Preview::Actions::CONFIRM_CANCEL_PURCHASE_UNINSTALLED) != 0) {
-            return new CancellingPurchasePreview(result, client, ppackage, manager, false);
+            return new CancellingPurchasePreview(result, metadata, client, ppackage, manager, false);
         } else if (metadict.count(click::Preview::Actions::CONFIRM_CANCEL_PURCHASE_INSTALLED) != 0) {
-            return new CancellingPurchasePreview(result, client, ppackage, manager, true);
+            return new CancellingPurchasePreview(result, metadata, client, ppackage, manager, true);
         } else if (metadict.count(click::Preview::Actions::RATED) != 0) {
             return new InstalledPreview(result, metadata, client, ppackage, depts);
         } else if (metadict.count(click::Preview::Actions::SHOW_UNINSTALLED) != 0) {
-            return new UninstalledPreview(result, client, depts, manager, ppackage);
+            return new UninstalledPreview(result, metadata, client, depts, manager, ppackage);
         } else {
             qWarning() << "preview() called with unexpected metadata. returning uninstalled preview";
-            return new UninstalledPreview(result, client, depts, manager, ppackage);
+            return new UninstalledPreview(result, metadata, client, depts, manager, ppackage);
         }
     } else {
         // metadata.scope_data() is Null, so we return an appropriate "default" preview:
@@ -246,7 +246,7 @@
         if (result["installed"].get_bool() == true) {
             return new InstalledPreview(result, metadata, client, ppackage, depts);
         } else {
-            return new UninstalledPreview(result, client, depts, manager, ppackage);
+            return new UninstalledPreview(result, metadata, client, depts, manager, ppackage);
         }
     }
 
@@ -410,7 +410,7 @@
 // to decide whether to show error widgets. see bug LP: #1289541
 void PreviewStrategy::populateDetails(std::function<void(const click::PackageDetails& details)> details_callback,
                               std::function<void(const click::ReviewList&,
-                                                    click::Reviews::Error)> reviews_callback)
+                                                    click::Reviews::Error)> reviews_callback, bool force_cache)
 {
 
     std::string app_name = get_string_maybe_null(result["name"]);
@@ -429,9 +429,9 @@
         // I think this should not be required when we switch the click::Index over
         // to using the Qt bridge. With that, the qt dependency becomes an implementation detail
         // and code using it does not need to worry about threading/event loop topics.
-        run_under_qt([this, details_callback, reviews_callback, app_name]()
+        run_under_qt([this, details_callback, reviews_callback, app_name, force_cache]()
             {
-                index_operation = index->get_details(app_name, [this, app_name, details_callback, reviews_callback](PackageDetails details, click::Index::Error error){
+                index_operation = index->get_details(app_name, [this, app_name, details_callback, reviews_callback, force_cache](PackageDetails details, click::Index::Error error){
                     if(error == click::Index::Error::NoError) {
                         qDebug() << "Got details:" << app_name.c_str();
                         details_callback(details);
@@ -445,8 +445,9 @@
                         details_callback(details);
                     }
                     reviews_operation = reviews->fetch_reviews(app_name,
-                                                               reviews_callback);
-                });
+                                                               reviews_callback,
+                                                               force_cache);
+                }, force_cache);
             });
     }
 }
@@ -842,6 +843,9 @@
 
 void InstalledPreview::run(unity::scopes::PreviewReplyProxy const& reply)
 {
+    const bool force_cache = (metadata.internet_connectivity() == scopes::QueryMetadata::ConnectivityStatus::Disconnected);
+    qDebug() << "preview, force_cache=" << force_cache << ", conn status=" << (int)metadata.internet_connectivity();
+
     // Check if the user is submitting a rating, so we can submit it.
     Review review;
     review.rating = 0;
@@ -911,7 +915,7 @@
             submit_future.get();
         }
     }
-    getApplicationUri(manifest, [this, reply, manifest, app_name, &review, userid](const std::string& uri) {
+    getApplicationUri(manifest, [this, reply, manifest, app_name, &review, userid, force_cache](const std::string& uri) {
             populateDetails([this, reply, uri, manifest, app_name](const PackageDetails &details){
                 cachedDetails = details;
                 store_department(details);
@@ -954,7 +958,7 @@
                 }
                 cachedWidgets.flush(reply);
                 reply->finished();
-        });
+        }, force_cache);
     });
 }
 
@@ -1207,12 +1211,14 @@
 // class UninstalledPreview
 
 UninstalledPreview::UninstalledPreview(const unity::scopes::Result& result,
+                                       const unity::scopes::ActionMetadata& metadata,
                                        const QSharedPointer<click::web::Client>& client,
                                        const std::shared_ptr<click::DepartmentsDb>& depts,
                                        const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                                        const QSharedPointer<pay::Package>& ppackage)
     : PreviewStrategy(result, client, ppackage),
       DepartmentUpdater(depts),
+      metadata(metadata),
       dm(new DownloadManager(client, manager))
 {
     qDebug() << "Creating new UninstalledPreview for result" << QString::fromStdString(result["name"].get_string());
@@ -1224,6 +1230,9 @@
 
 void UninstalledPreview::run(unity::scopes::PreviewReplyProxy const& reply)
 {
+    const bool force_cache = (metadata.internet_connectivity() == scopes::QueryMetadata::ConnectivityStatus::Disconnected);
+    qDebug() << "preview, force_cache=" << force_cache << ", conn status=" << (int)metadata.internet_connectivity();
+
     qDebug() << "in UninstalledPreview::run, about to populate details";
     populateDetails([this, reply](const PackageDetails &details){
             store_department(details);
@@ -1257,7 +1266,7 @@
                 reply->finished();
                 qDebug() << "---------- Finished reply for:" << result["name"].get_string().c_str();
             });
-        });
+        }, force_cache);
 }
 
 scopes::PreviewWidgetList UninstalledPreview::uninstalledActionButtonWidgets(const PackageDetails &details)
@@ -1310,10 +1319,11 @@
 
 // TODO: this class should be removed once uninstall() is handled elsewhere.
 UninstallingPreview::UninstallingPreview(const unity::scopes::Result& result,
+                                         const unity::scopes::ActionMetadata& metadata,
                                          const QSharedPointer<click::web::Client>& client,
                                          const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                                          const QSharedPointer<pay::Package>& ppackage)
-    : UninstalledPreview(result, client, nullptr, manager, ppackage)
+    : UninstalledPreview(result, metadata, client, nullptr, manager, ppackage)
 {
 }
 
@@ -1353,11 +1363,12 @@
 // class CancellingPurchasePreview : public UninstallingPreview
 
 CancellingPurchasePreview::CancellingPurchasePreview(const unity::scopes::Result& result,
+                                                     const unity::scopes::ActionMetadata& metadata,
                                                      const QSharedPointer<click::web::Client>& client,
                                                      const QSharedPointer<pay::Package>& ppackage,
                                                      const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                                                      bool installed)
-    : UninstallingPreview(result, client, manager, ppackage),
+    : UninstallingPreview(result, metadata, client, manager, ppackage),
       installed(installed)
 {
 }

=== modified file 'libclickscope/click/preview.h'
--- libclickscope/click/preview.h	2016-03-07 14:10:27 +0000
+++ libclickscope/click/preview.h	2016-05-06 14:29:14 +0000
@@ -177,7 +177,8 @@
 protected:
     virtual void populateDetails(std::function<void(const PackageDetails &)> details_callback,
                                  std::function<void(const click::ReviewList&,
-                                                    click::Reviews::Error)> reviews_callback);
+                                                    click::Reviews::Error)> reviews_callback,
+                                 bool force_cache = false);
     virtual scopes::PreviewWidgetList headerWidgets(const PackageDetails &details);
     virtual scopes::PreviewWidgetList screenshotsWidgets(const PackageDetails &details);
     virtual scopes::PreviewWidgetList descriptionWidgets(const PackageDetails &details);
@@ -317,6 +318,7 @@
 {
 public:
     UninstalledPreview(const unity::scopes::Result& result,
+                       const unity::scopes::ActionMetadata& metadata,
                        const QSharedPointer<click::web::Client>& client,
                        const std::shared_ptr<click::DepartmentsDb>& depts,
                        const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
@@ -326,6 +328,7 @@
 
     void run(unity::scopes::PreviewReplyProxy const& reply) override;
 protected:
+    scopes::ActionMetadata metadata;
     PackageDetails found_details;
     CachedPreviewWidgets cachedWidgets;
     std::string found_object_path;
@@ -340,6 +343,7 @@
 {
 public:
     UninstallingPreview(const unity::scopes::Result& result,
+                        const unity::scopes::ActionMetadata& metadata,
                         const QSharedPointer<click::web::Client>& client,
                         const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                         const QSharedPointer<pay::Package>& ppackage);
@@ -357,6 +361,7 @@
 {
 public:
     CancellingPurchasePreview(const unity::scopes::Result& result,
+                              const unity::scopes::ActionMetadata& metadata,
                               const QSharedPointer<click::web::Client>& client,
                               const QSharedPointer<pay::Package>& ppackage,
                               const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,

=== modified file 'libclickscope/click/reviews.cpp'
--- libclickscope/click/reviews.cpp	2016-04-01 17:16:05 +0000
+++ libclickscope/click/reviews.cpp	2016-05-06 14:29:14 +0000
@@ -126,13 +126,14 @@
 }
 
 click::web::Cancellable Reviews::fetch_reviews (const std::string& package_name,
-                                                std::function<void(ReviewList, Error)> callback)
+                                                std::function<void(ReviewList, Error)> callback,
+                                                bool force_cache)
 {
     click::web::CallParams params;
     params.add(click::REVIEWS_QUERY_ARGNAME, package_name.c_str());
     QSharedPointer<click::web::Response> response = client->call
         (get_base_url() + click::REVIEWS_API_PATH, "GET", false,
-         std::map<std::string, std::string>{}, "", params);
+         std::map<std::string, std::string>{}, "", params, force_cache);
 
     QObject::connect(response.data(), &click::web::Response::finished,
                 [=](QString reply) {

=== modified file 'libclickscope/click/reviews.h'
--- libclickscope/click/reviews.h	2015-11-24 18:23:23 +0000
+++ libclickscope/click/reviews.h	2016-05-06 14:29:14 +0000
@@ -78,7 +78,8 @@
     virtual ~Reviews();
 
     virtual click::web::Cancellable fetch_reviews (const std::string& package_name,
-                                           std::function<void(ReviewList, Error)> callback);
+                                           std::function<void(ReviewList, Error)> callback,
+                                           bool force_cache = false);
     click::web::Cancellable submit_review (const Review& review,
                                            std::function<void(Error)> callback);
     click::web::Cancellable edit_review (const Review& review,

=== modified file 'libclickscope/click/webclient.cpp'
--- libclickscope/click/webclient.cpp	2016-04-01 17:16:05 +0000
+++ libclickscope/click/webclient.cpp	2016-05-06 14:29:14 +0000
@@ -80,10 +80,11 @@
 
 QSharedPointer<click::web::Response> click::web::Client::call(
     const std::string& iri,
-    const click::web::CallParams& params)
+    const click::web::CallParams& params,
+    bool force_cache)
 {
     return call(iri, "GET", true,
-                std::map<std::string, std::string>(), "", params);
+                std::map<std::string, std::string>(), "", params, force_cache);
 }
 
 QSharedPointer<click::web::Response> click::web::Client::call(
@@ -92,7 +93,8 @@
     bool sign,
     const std::map<std::string, std::string>& headers,
     const std::string& data,
-    const click::web::CallParams& params)
+    const click::web::CallParams& params,
+    bool force_cache)
 {
     QUrl url(iri.c_str());
     url.setQuery(params.query);
@@ -104,7 +106,11 @@
     request->setRawHeader(ACCEPT_LANGUAGE_HEADER.c_str(),
                           Configuration().get_accept_languages().c_str());
 
-    request->setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache);
+    if (force_cache) {
+        qDebug() << "Forcing cache";
+    }
+    request->setAttribute(QNetworkRequest::CacheLoadControlAttribute, force_cache ? QNetworkRequest::AlwaysCache : QNetworkRequest::PreferCache);
+    request->setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true);
 
     for (const auto& kv : headers) {
         QByteArray header_name(kv.first.c_str(), kv.first.length());

=== modified file 'libclickscope/click/webclient.h'
--- libclickscope/click/webclient.h	2016-04-01 17:16:05 +0000
+++ libclickscope/click/webclient.h	2016-05-06 14:29:14 +0000
@@ -117,14 +117,14 @@
 
     virtual QSharedPointer<Response> call(
         const std::string& iri,
-        const CallParams& params = CallParams());
+        const CallParams& params = CallParams(), bool force_cache = false);
     virtual QSharedPointer<Response> call(
         const std::string& iri,
         const std::string& method,
         bool sign = true,
         const std::map<std::string, std::string>& headers = std::map<std::string, std::string>(),
         const std::string& data = "",
-        const CallParams& params = CallParams());
+        const CallParams& params = CallParams(), bool force_cache = false);
     void setCredentialsService(const QSharedPointer<click::CredentialsService>& sso);
     virtual void invalidateCredentials();
 

=== modified file 'libclickscope/tests/mock_webclient.h'
--- libclickscope/tests/mock_webclient.h	2016-04-01 17:16:05 +0000
+++ libclickscope/tests/mock_webclient.h	2016-05-06 14:29:14 +0000
@@ -86,7 +86,7 @@
         const click::web::CallParams& params));
     QSharedPointer<click::web::Response> call(
         const std::string& iri,
-        const click::web::CallParams& params=click::web::CallParams()) override {
+        const click::web::CallParams& params=click::web::CallParams(), bool = false) override {
         return callImpl(iri, "GET", true,
                         std::map<std::string, std::string>(), "", params);
     }
@@ -96,7 +96,7 @@
         bool sign = true,
         const std::map<std::string, std::string>& headers = std::map<std::string, std::string>(),
         const std::string& data = "",
-        const click::web::CallParams& params=click::web::CallParams()) override {
+        const click::web::CallParams& params=click::web::CallParams(), bool = false) override {
         return callImpl(iri, method, sign, headers, data, params);
     }
 

=== modified file 'libclickscope/tests/test_preview.cpp'
--- libclickscope/tests/test_preview.cpp	2016-02-24 17:50:03 +0000
+++ libclickscope/tests/test_preview.cpp	2016-05-06 14:29:14 +0000
@@ -64,7 +64,7 @@
     FakeIndex() {
 
     }
-    click::web::Cancellable get_details(const std::string& /*package_name*/, std::function<void(click::PackageDetails, Error)> callback) override {
+    click::web::Cancellable get_details(const std::string& /*package_name*/, std::function<void(click::PackageDetails, Error)> callback, bool) override {
         callback(click::PackageDetails(), Error::NetworkError);
         return click::web::Cancellable();
     }
@@ -78,7 +78,7 @@
 
     }
 
-    click::web::Cancellable fetch_reviews(const std::string &/*package_name*/, std::function<void (click::ReviewList, Error)> callback) override {
+    click::web::Cancellable fetch_reviews(const std::string &/*package_name*/, std::function<void (click::ReviewList, Error)> callback, bool) override {
         callback(click::ReviewList(), Error::NoError);
         return click::web::Cancellable();
     }
@@ -406,11 +406,12 @@
 public:
     FakeBaseUninstalledPreview(const std::string& object_path,
                                const unity::scopes::Result& result,
+                               const unity::scopes::ActionMetadata& metadata,
                                const QSharedPointer<click::web::Client>& client,
                                const std::shared_ptr<click::DepartmentsDb>& depts,
                                const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                                const QSharedPointer<pay::Package> pay_package)
-        : click::UninstalledPreview(result, client, depts, manager, pay_package),
+        : click::UninstalledPreview(result, metadata, client, depts, manager, pay_package),
           object_path(object_path)
     {
     }
@@ -429,11 +430,12 @@
     MOCK_METHOD1(progressBarWidget, scopes::PreviewWidgetList(const std::string& object_path));
     FakeUninstalledPreview(const std::string& object_path,
                            const unity::scopes::Result& result,
+                           const unity::scopes::ActionMetadata& metadata,
                            const QSharedPointer<click::web::Client>& client,
                            const std::shared_ptr<click::DepartmentsDb>& depts,
                            const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                            const QSharedPointer<pay::Package> pay_package)
-        : FakeBaseUninstalledPreview(object_path, result, client, depts, manager, pay_package) {
+        : FakeBaseUninstalledPreview(object_path, result, metadata, client, depts, manager, pay_package) {
     }
 };
 
@@ -444,7 +446,8 @@
 
     result["name"] = "fake_app_name";
     scopes::PreviewWidgetList response;
-    FakeUninstalledPreview preview(fake_object_path, result, client, depts, sdm, pay_package);
+    unity::scopes::ActionMetadata metadata("en_EN", "desktop");
+    FakeUninstalledPreview preview(fake_object_path, result, metadata, client, depts, sdm, pay_package);
     EXPECT_CALL(preview, progressBarWidget(_))
             .Times(1)
             .WillOnce(Return(response));
@@ -458,7 +461,8 @@
 
     result["name"] = "fake_app_name";
     scopes::PreviewWidgetList response;
-    FakeUninstalledPreview preview(fake_object_path, result, client, depts, sdm, pay_package);
+    unity::scopes::ActionMetadata metadata("en_EN", "desktop");
+    FakeUninstalledPreview preview(fake_object_path, result, metadata, client, depts, sdm, pay_package);
     EXPECT_CALL(preview, uninstalledActionButtonWidgets(_))
             .Times(1)
             .WillOnce(Return(response));
@@ -468,11 +472,12 @@
 class FakeUninstalledRefundablePreview : FakeBaseUninstalledPreview {
 public:
     FakeUninstalledRefundablePreview(const unity::scopes::Result& result,
+                                     const unity::scopes::ActionMetadata& metadata,
                                      const QSharedPointer<click::web::Client>& client,
                                      const std::shared_ptr<click::DepartmentsDb>& depts,
                                      const QSharedPointer<Ubuntu::DownloadManager::Manager>& manager,
                                      const QSharedPointer<pay::Package> pay_package)
-        : FakeBaseUninstalledPreview(std::string{""}, result, client, depts, manager, pay_package){
+        : FakeBaseUninstalledPreview(std::string{""}, result, metadata, client, depts, manager, pay_package){
     }
     using click::UninstalledPreview::uninstalledActionButtonWidgets;
     MOCK_METHOD0(isRefundable, bool());
@@ -493,7 +498,8 @@
     result["name"] = "fake_app_name";
     result["price"] = 2.99;
     result["purchased"] = true;
-    FakeUninstalledRefundablePreview preview(result, client, depts, sdm, pay_package);
+    unity::scopes::ActionMetadata metadata("en_EN", "desktop");
+    FakeUninstalledRefundablePreview preview(result, metadata, client, depts, sdm, pay_package);
 
     click::PackageDetails pkgdetails;
     EXPECT_CALL(preview, isRefundable()).Times(1)
@@ -506,7 +512,8 @@
     result["name"] = "fake_app_name";
     result["price"] = 2.99;
     result["purchased"] = true;
-    FakeUninstalledRefundablePreview preview(result, client, depts, sdm, pay_package);
+    unity::scopes::ActionMetadata metadata("en_EN", "desktop");
+    FakeUninstalledRefundablePreview preview(result, metadata, client, depts, sdm, pay_package);
 
     click::PackageDetails pkgdetails;
     EXPECT_CALL(preview, isRefundable()).Times(1)

=== modified file 'scope/clickstore/store-query.cpp'
--- scope/clickstore/store-query.cpp	2016-01-12 10:23:19 +0000
+++ scope/clickstore/store-query.cpp	2016-05-06 14:29:14 +0000
@@ -46,7 +46,6 @@
 #include <unity/scopes/Variant.h>
 #include <unity/scopes/VariantBuilder.h>
 
-#include <iostream>
 #include <iomanip>
 #include<vector>
 #include<set>
@@ -533,11 +532,14 @@
                 this->finished(searchReply); //FIXME: this shouldn't be needed
             };
 
+            const bool force_cache = (search_metadata().internet_connectivity() == scopes::QueryMetadata::ConnectivityStatus::Disconnected);
+            qDebug() << "search, force_cache=" << force_cache << ", conn status=" << (int)search_metadata().internet_connectivity();
+
             // this is the case when we do bootstrap for the first time, or it failed last time
             if (impl->department_lookup.size() == 0)
             {
                 qDebug() << "performing bootstrap request";
-                impl->search_operation = impl->index.bootstrap([this, search_cb, searchReply, installedPackages](const DepartmentList& deps, const
+                impl->search_operation = impl->index.bootstrap([this, search_cb, searchReply, installedPackages, force_cache](const DepartmentList& deps, const
                             HighlightList& highlights, click::Index::Error error, int) {
                 if (error == click::Index::Error::NoError)
                 {
@@ -574,10 +576,10 @@
                     {
                         qDebug() << "starting search of" << QString::fromStdString(query().query_string());
                         push_departments(searchReply);
-                        impl->search_operation = impl->index.search(query().query_string(), query().department_id(), search_cb);
+                        impl->search_operation = impl->index.search(query().query_string(), query().department_id(), search_cb, force_cache);
                     }
                 }
-            });
+            }, force_cache);
         }
         else
         {
@@ -589,7 +591,7 @@
             {
                 qDebug() << "starting search of" << QString::fromStdString(query().query_string());
                 push_departments(searchReply);
-                impl->search_operation = impl->index.search(query().query_string(), query().department_id(), search_cb);
+                impl->search_operation = impl->index.search(query().query_string(), query().department_id(), search_cb, force_cache);
             }
         }
     });

=== modified file 'scope/tests/test_helpers.h'
--- scope/tests/test_helpers.h	2015-12-01 15:39:41 +0000
+++ scope/tests/test_helpers.h	2016-05-06 14:29:14 +0000
@@ -53,14 +53,15 @@
 
     }
 
-    click::web::Cancellable search(const std::string &query, const std::string &department, std::function<void (click::Packages, click::Packages)> callback) override
+    click::web::Cancellable search(const std::string &query, const std::string &department, std::function<void (click::Packages, click::Packages)> callback,
+            bool) override
     {
         do_search(query, department, callback);
         callback(packages, recommends);
         return click::web::Cancellable();
     }
 
-    click::web::Cancellable bootstrap(std::function<void(const click::DepartmentList&, const click::HighlightList&, Error, int)> callback) override
+    click::web::Cancellable bootstrap(std::function<void(const click::DepartmentList&, const click::HighlightList&, Error, int)> callback, bool) override
     {
         callback(bootstrap_departments, bootstrap_highlights, click::Index::Error::NoError, 0);
         return click::web::Cancellable();