Merge lp:~canonical-platform-qa/qakit/test-execution-gathering into lp:qakit

[Allan LeSage]

Gather test execution data from PractiTest.

Gather test execution data from PractiTest and deposit in a mongodb database.

[Christopher Lee (veebers)]

Looking good, a couple of inline comments.

review: Needs Fixing

[lp:~canonical-platform-qa/qakit/test-execution-gathering]

25. By Allan LeSage on 2015-10-29

De-shebangify.

26. By Allan LeSage on 2015-10-29

Adjust count insertion.

27. By Allan LeSage on 2015-10-29

Adjust a warning log message.

28. By Allan LeSage on 2015-10-29

Adjust a docstring to call out list.

29. By Allan LeSage on 2015-10-29

Move no last testset logic up a level, some whitespace and naming changes thrown in.

30. By Allan LeSage on 2015-10-29

Resolve flake8 config conflict, remove NOQA.

31. By Allan LeSage on 2015-10-29

Just raise.

[Allan LeSage (allanlesage)]

Addressed in numbered revisions, thanks Chris.

[Christopher Lee (veebers)]

LGTM, thanks for the changes.

review: Approve

=== added directory 'qakit/metrics/practitest'
=== added file 'qakit/metrics/practitest/instances.py'
--- qakit/metrics/practitest/instances.py	1970-01-01 00:00:00 +0000
+++ qakit/metrics/practitest/instances.py	2015-10-29 22:05:12 +0000
@@ -0,0 +1,102 @@
+# UESQA Metrics
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+
+import qakit.practitest.practitest as practitest
+
+logger = logging.getLogger(__name__)
+
+
+def append_testset_id(instance):
+    """Append TestSet ID to the given PractiTest instance dict.
+
+    This makes querying much easier later, else it has to be parsed
+    out of a string.
+
+    :param instance: a PractiTest instance
+
+    """
+    instance['testset_id'] = int(instance['id'].split(':')[0])
+    return instance
+
+
+def append_data_to_instance_dict(instance):
+    """Append TestSet ID and last run datetime to the given instance.
+
+    :param instance: a PractiTest instance to which to append data
+
+    """
+    instance = practitest.append_last_run_datetime(instance)
+    instance = append_testset_id(instance)
+    return instance
+
+
+def update_testset_instances(db, session, testset_id):
+    """Retrieve instances from PractiTest for the given TestSet ID.
+
+    NOTE that old runs get crushed on the PractiTest side: e.g. if a
+    test fails and it's re-run, the failure is lost to the API.
+
+    :param db: MongoClient db in which to store PractiTest results
+    :param practitest_session: PractitestSession
+    :param testset_id: the PractiTest TestSet for which to retrieve instances.
+
+    """
+    logger.info("Retrieving instances for TestSet {}.".format(testset_id))
+    try:
+        instances = session.get_instances(testset_id)
+    except ValueError as e:
+        logger.warn('TestSet not found for id: {}: '.format(
+            testset_id), str(e))
+        return 0
+    count = 0
+    for instance in instances:
+        instance = append_data_to_instance_dict(instance)
+        result = db.instances.update({'system_id': instance['system_id']},
+                                     instance,
+                                     upsert=True)
+        count += result['n']
+    logger.info("Retrieved {} instances for TestSet {}.".format(count,
+                                                                testset_id))
+    return count
+
+
+def update_testsets_instances(db, practitest_session, testset_ids=None):
+    """Update instances for the given testsets, finding new instances.
+
+    An instance represents a test execution in PractiTest.  Note that if a
+    test is executed more than once, only the last execution is reported via
+    the API: e.g. if a test fails and then is run again and passes, the
+    failure is lost to us.
+
+    :param db: a MongoClient db in which to store PractiTest instances
+    :param practitest_session: a PractitestSession
+    :param testset_ids: list of PractiTest TestSet ids for which to get
+        instances: numbers appear in header of TestSet webpage.  If no ids are
+        given, retrieve instances for all *known* TestSets (i.e. all already
+        in the given database).
+
+    """
+    logger.info("Updating TestSets with new instance runs.")
+    if testset_ids is None:
+        testset_ids = [testset['id'] for testset in db.testsets.find()]
+    count = 0
+    for testset_id in testset_ids:
+        count += update_testset_instances(
+            db, practitest_session, testset_id)
+    logger.info("Updated {} instances.".format(count))

=== added file 'qakit/metrics/practitest/testsets.py'
--- qakit/metrics/practitest/testsets.py	1970-01-01 00:00:00 +0000
+++ qakit/metrics/practitest/testsets.py	2015-10-29 22:05:12 +0000
@@ -0,0 +1,144 @@
+# UESQA Metrics
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import pymongo
+from qakit.practitest.practitest import append_last_run_datetime
+
+logger = logging.getLogger(__name__)
+
+
+def retrieve_testset(db, practitest_session, testset_id):
+    """Retrieve a testset of the given ID, storing it in our database.
+
+    :param db: a MongoClient collection in which testsets are stored
+    :param practitest_session: a PractiTestSession
+    :param testset_id: the ID of the PractiTest testset to retrieve
+    :raises ValueError: if testset_id is not found in PractiTest
+
+    """
+    try:
+        testset = practitest_session.get_testset(testset_id)
+    except ValueError:
+        logger.warn("Testset {} not found!".format(testset_id))
+        raise
+    testset = append_last_run_datetime(testset)
+    result = db.testsets.update(
+        {'system_id': testset['system_id']},
+        testset,
+        upsert=True)
+    if result['updatedExisting']:
+        logger.info("Updated testset {}.".format(testset_id))
+    else:
+        logger.info("Inserted testset {}.".format(testset_id))
+    return result
+
+
+def _get_last_known_testset_id(db):
+    """Get the last known testset ID from a db.
+
+    :param db: a MongoClient collection in which testsets are stored
+
+    """
+    testsets = db.testsets.find()
+    try:
+        return testsets.sort('id', pymongo.DESCENDING).limit(1)[0]['id']
+    except IndexError:
+        raise ValueError("No testsets found in db.")
+
+
+def update_testsets(db, practitest_session, testset_ids=None):
+    """Update PractiTest testsets with new data.
+
+    Note that we update testsets themselves only--instances (i.e. test
+    executions) should be updated separately.
+
+    :param db: a MongoClient collection in which testsets are stored
+    :param practitest_session: a PractiTestSession
+    :param testset_ids: a list of testsets to retrieve; if not specified,
+        all known testsets in the given db are updated
+
+    """
+    logger.info("Updating existing testsets.")
+    if testset_ids is None:
+        testset_ids = [testset['id'] for testset in db.testsets.find()]
+    count = 0
+    for testset_id in testset_ids:
+        try:
+            result = retrieve_testset(db, practitest_session, testset_id)
+            count += result['n']
+        except ValueError:
+            logger.warn(
+                "Testset {} not found, assume deleted.".format(testset_id))
+    logger.info("Updated {} testsets.".format(count))
+
+
+def _scan_for_testsets(db, practitest_session, testset_id_range):
+    """Scan a range of testset IDs, retrieving each if it exists in PractiTest.
+
+    Use this to scout ahead while discovering new testsets.
+
+    :param db: a MongoClient collection in which testsets are stored
+    :param practitest_session: a PractiTestSession
+    :param testset_id_range: a range of testset IDs
+
+    """
+    count = 0
+    for testset_id in testset_id_range:
+        try:
+            result = retrieve_testset(db, practitest_session, testset_id)
+            count += result['n']
+        except ValueError:
+            logger.debug("Testset {} not found.".format(testset_id))
+    return count
+
+
+def discover_new_testsets(db, practitest_session, increment=50):
+    """Discover and retrieve new testsets in PractiTest.
+
+    PractiTest API won't give a list of IDs of testsets belonging to a
+    project.  IDs are assigned incrementally and occasionally deleted;
+    scan a given number of IDs ahead and attempt to find a testset at
+    each ID, continue until a full scan comes up empty.
+
+    :param db: a MongoClient collection in which testsets are stored
+    :param practitest_session: a PractiTestSession
+    :param increment: the increment by which to scan ahead, defaults to 50
+
+    """
+    logger.info("Discovering new testsets in PractiTest.")
+    try:
+        logger.info("Last known testset is {}.".format(last_testset_id))
+        last_testset_id = _get_last_known_testset_id(db)
+    except ValueError as e:
+        logger.info("No testsets in db.")
+        last_testset_id = 0
+    total_count = 0
+    testsets_found = True
+    next_testset_id = last_testset_id + 1
+    while testsets_found:
+        testset_id_range = range(next_testset_id, next_testset_id+increment)
+        scan_count = _scan_for_testsets(
+            db,
+            practitest_session,
+            testset_id_range)
+        if scan_count is 0:
+            testsets_found = False
+        else:
+            next_testset_id = testset_id_range[-1] + 1
+        total_count += scan_count
+    logger.info("Retrieved {} testsets.".format(total_count))
+    return total_count

=== added file 'qakit/metrics/test_execution.py'
--- qakit/metrics/test_execution.py	1970-01-01 00:00:00 +0000
+++ qakit/metrics/test_execution.py	2015-10-29 22:05:12 +0000
@@ -0,0 +1,131 @@
+#!/usr/bin/python3
+# UESQA Metrics
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import argparse
+import configparser
+import logging
+import sys
+
+import pymongo
+
+import qakit.config as qakit_config
+import qakit.metrics.practitest.instances as instances
+import qakit.metrics.practitest.testsets as testsets
+import qakit.metrics.util as util
+import qakit.practitest.practitest as practitest
+
+logger = logging.getLogger(__name__)
+
+
+def update_testsets(db, practitest_session, testset_ids=None):
+    """Discover new and update existing PractiTest testsets.
+
+    :param db: a MongoClient collection in which PractiTest data is stored
+    :param practitest_session: a PractiTestSession
+    :param testsets: a list of PractiTest testset ids to update; if not
+        specified, new testsets are discovered and all existing testsets are
+        updated
+
+    """
+    if not testset_ids:
+        testset_ids = [testset['id'] for testset in db.testsets.find()]
+        # (newly-discovered testsets will be updated upon discovery)
+        testsets.discover_new_testsets(db, practitest_session)
+    testsets.update_testsets(db, practitest_session, testset_ids=testset_ids)
+
+
+def update_instances(db, practitest_session, testset_ids=None):
+    """Discover new and update existing instances for the given testset IDs.
+
+    :param db: a MongoClient collection in which PractiTest data is stored
+    :param practitest_session: a PractiTestSession
+    :param testsets: a list of PractiTest testset ids to update; if not
+        specified, all known testsets are updated
+
+    """
+    if not testset_ids:
+        testset_ids = [testset['id'] for testset in db.testsets.find()]
+    instances.update_testsets_instances(
+        db,
+        practitest_session,
+        testset_ids=testset_ids)
+
+
+def update_test_execution_data(db, practitest_session, testset_ids=None):
+    """Update PractiTest test execution data: testsets and instances.
+
+    :param db: a MongoClient collection in which PractiTest data is stored
+    :param practitest_session: a PractiTestSession
+    :param testsets: a list of PractiTest testset ids to update; if not
+        specified, new testsets are discovered and all existing testsets are
+        updated
+
+    """
+    update_testsets(db, practitest_session, testset_ids=testset_ids)
+    update_instances(db, practitest_session, testset_ids=testset_ids)
+
+
+def _read_config(config_filepath):
+    """Parse the config at the given filepath, returning a config dict.
+
+    :param config_filepath: the filepath to a configuration file
+
+    """
+    config_file = configparser.ConfigParser()
+    config_file.read(config_filepath)
+    config = {}
+    for var_name in ('PRACTITEST_PROJECT_ID',
+                     'PRACTITEST_API_KEY',
+                     'PRACTITEST_API_SECRET_KEY'):
+        new_var_name = var_name.lower().replace('practitest_', '')
+        config[new_var_name] = config_file['DEFAULT'][var_name]
+    return config
+
+
+def _parse_arguments():
+    parser = argparse.ArgumentParser(
+        "Retrieve PractiTest results, storing them in a MongoDB database.")
+    parser.add_argument(
+        nargs='*',
+        dest='testsets',
+        help='TestSet from which to retrieve instances',
+        type=int)
+    return parser.parse_args()
+
+
+def main():
+    util.setup_logging()
+    config_dict = _read_config(qakit_config.get_config_file_location())
+    args = _parse_arguments()
+    conn = pymongo.MongoClient()
+    db = conn.metrics
+    practitest_session = practitest.PractitestSession(
+        config_dict['project_id'],
+        config_dict['api_key'],
+        config_dict['api_secret_key'])
+    update_test_execution_data(
+        db,
+        practitest_session,
+        testset_ids=args.testsets)
+
+
+if __name__ == '__main__':
+    try:
+        sys.exit(main())
+    except Exception as e:
+        logger.error(e, exc_info=True)

=== added file 'qakit/metrics/tests/test_practitest_instances.py'
--- qakit/metrics/tests/test_practitest_instances.py	1970-01-01 00:00:00 +0000
+++ qakit/metrics/tests/test_practitest_instances.py	2015-10-29 22:05:12 +0000
@@ -0,0 +1,113 @@
+# UESQA Metrics
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from unittest import mock
+
+from bson import ObjectId
+
+import qakit.metrics.practitest.instances as instances
+
+
+INSTANCE = {
+    "_id": ObjectId("5590253dca21169c04a38b51"),
+    "tester": {
+        "value": "iahmad",
+        "name": "Tester"
+    },
+    "last_run_week_number": 20,
+    "name": "Complete Edges Intro",
+    "last_run": {
+        "value": "21-May-2015  20:59",
+        "name": "Last Run"
+    },
+    "___f_10578": {
+        "value": None,
+        "name": "Build #"
+    },
+    "id": "10:1",
+    "system_id": 1393092,
+    "run_status": {
+        "value": "FAILED",
+        "name": "Run Status"
+    }
+}
+
+
+TESTSETS = [
+    {'id': 761},
+    {'id': 762},
+    {'id': 763},
+    {'id': 764},
+    {'id': 765},
+    {'id': 766},
+    {'id': 767}
+]
+
+
+class AppendTestSetIdTestCase(unittest.TestCase):
+
+    def test_testset_id_appended_correctly(self):
+        instance = instances.append_testset_id(INSTANCE)
+        self.assertEqual(10, instance['testset_id'])
+
+
+class AppendDataToInstanceDictTestCase(unittest.TestCase):
+
+    @mock.patch('qakit.practitest.practitest.append_last_run_datetime')
+    @mock.patch('qakit.metrics.practitest.instances.append_testset_id')
+    def test_appends_called(self,
+                            mock_append_testset_id,
+                            mock_append_last_run_datetime):
+        # I agree, this should never fail :)
+        mock_append_last_run_datetime.return_value = 'fake-instance'
+        instances.append_data_to_instance_dict(INSTANCE)
+        mock_append_last_run_datetime.assert_called_with(INSTANCE)
+        mock_append_testset_id.assert_called_with('fake-instance')
+
+
[e-mail]('qakit.metrics.practitest.instances.update_testset_instances')
+class UpdateTestSetInstancesTestCase(unittest.TestCase):
+
+    def test_no_testset_ids_specified(self, mock_update_testset_instances):
+        db = mock.Mock(
+            testsets=mock.Mock(
+                find=mock.Mock(return_value=TESTSETS)))
+        instances.update_testsets_instances(
+            db,
+            'fake-practitest-session',
+            None)
+        self.assertEqual(7, mock_update_testset_instances.call_count)
+
+    def test_one_testset_specified(self, mock_update_testset_instances):
+        instances.update_testsets_instances(
+            'fake-db',
+            'fake-practitest-session',
+            [761])
+        self.assertEqual(1, mock_update_testset_instances.call_count)
+        mock_update_testset_instances.assert_called_with(
+            'fake-db', 'fake-practitest-session', 761)
+
+    def test_several_testsets_specified(self, mock_update_testset_instances):
+        testset_ids = [761, 762, 763]
+        instances.update_testsets_instances(
+            'fake-db',
+            'fake-practitest-session',
+            testset_ids)
+        self.assertEqual(3, mock_update_testset_instances.call_count)
+        for testset_id in testset_ids:
+            mock_update_testset_instances.assert_any_call(
+                'fake-db', 'fake-practitest-session', testset_id)

=== added file 'qakit/metrics/tests/test_practitest_testsets.py'
--- qakit/metrics/tests/test_practitest_testsets.py	1970-01-01 00:00:00 +0000
+++ qakit/metrics/tests/test_practitest_testsets.py	2015-10-29 22:05:12 +0000
@@ -0,0 +1,80 @@
+# UESQA Metrics
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from unittest import mock
+
+import qakit.metrics.practitest.testsets as testsets
+
+TESTSETS = [
+    {'id': 761},
+    {'id': 762},
+    {'id': 763},
+    {'id': 764},
+    {'id': 765},
+    {'id': 766},
+    {'id': 767}
+]
+
+
+class GetLastKnownTestsetIdTestCase(unittest.TestCase):
+
+    def testset_exists(self):
+        db = mock.Mock(
+            testsets=mock.Mock(
+                find=mock.Mock(
+                    return_value=mock.Mock(
+                        sort=mock.Mock(
+                            return_value=mock.Mock(
+                                limit=mock.Mock(
+                                    return_value=TESTSETS[:1])))))))
+        self.assertEqual(
+            761,
+            testsets._get_last_known_testset_id(db))
+
+    def test_no_testsets_raises_valueerror(self):
+        testsets_ = mock.MagicMock()
+        get_item = testsets_.sort.return_value.limit.return_value.__getitem__
+        get_item.side_effect = IndexError
+        db = mock.Mock(
+            testsets=mock.Mock(
+                find=mock.Mock(
+                    return_value=testsets_)))
+        with self.assertRaises(ValueError):
+            testsets._get_last_known_testset_id(db)
+
+
[e-mail]('qakit.metrics.practitest.testsets.retrieve_testset')
+class UpdateTestsetsTestCase(unittest.TestCase):
+
+    def test_no_testsets_specified(self, mock_retrieve_testset):
+        db = mock.Mock(
+            testsets=mock.Mock(
+                find=mock.Mock(
+                    return_value=TESTSETS)))
+        mock_retrieve_testset.return_value = {'n': 1}
+        testsets.update_testsets(db, 'fake-practitest-session')
+        for testset_id in [testset['id'] for testset in TESTSETS]:
+            mock_retrieve_testset.assert_any_call(
+                db, 'fake-practitest-session', testset_id)
+
+    def test_testsets_specified(self, mock_retrieve_testset):
+        mock_retrieve_testset.return_value = {'n': 1}
+        testsets_ = [testset['id'] for testset in TESTSETS]
+        testsets.update_testsets('fake-db', 'fake-practitest-session', testsets_)
+        for testset_id in [testset['id'] for testset in TESTSETS]:
+            mock_retrieve_testset.assert_any_call(
+                'fake-db', 'fake-practitest-session', testset_id)

=== added file 'qakit/metrics/tests/test_test_execution.py'
--- qakit/metrics/tests/test_test_execution.py	1970-01-01 00:00:00 +0000
+++ qakit/metrics/tests/test_test_execution.py	2015-10-29 22:05:12 +0000
@@ -0,0 +1,99 @@
+# UESQA Metrics
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from unittest import mock
+
+import qakit.metrics.test_execution as test_execution
+
+
+TESTSETS = [
+    {'id': 761},
+    {'id': 762},
+    {'id': 763},
+    {'id': 764},
+    {'id': 765},
+    {'id': 766},
+    {'id': 767}
+]
+
+
[e-mail]('qakit.metrics.test_execution.update_testsets')
[e-mail]('qakit.metrics.test_execution.update_instances')
+class UpdateTestExecutionDataTestCase(unittest.TestCase):
+
+    def test_update_both_testsets_and_instances(
+            self,
+            mock_update_instances,
+            mock_update_testsets):
+        test_execution.update_test_execution_data(
+            'fake-db', 'fake-practitest-session', 'fake-testset-ids')
+        for f in mock_update_instances, mock_update_testsets:
+            f.assert_called_with(
+                'fake-db',
+                'fake-practitest-session',
+                testset_ids='fake-testset-ids')
+
+
[e-mail]('qakit.metrics.practitest.testsets.update_testsets')
+class UpdateTestSetsTestCase(unittest.TestCase):
+
+    def test_testset_ids_specified(self, mock_update_testsets):
+        test_execution.update_testsets(
+            'fake-db', 'fake-practitest-session', 'fake-testset-ids')
+        mock_update_testsets.assert_called_with(
+            'fake-db',
+            'fake-practitest-session',
+            testset_ids='fake-testset-ids')
+
+    @mock.patch('qakit.metrics.practitest.testsets.discover_new_testsets')
+    def test_no_testset_ids_specified(
+            self,
+            mock_discover_new_testsets,
+            mock_update_testsets):
+        db = mock.Mock(
+            testsets=mock.Mock(
+                find=mock.Mock(return_value=TESTSETS)))
+        test_execution.update_testsets(
+            db, 'fake-practitest-session', testset_ids=None)
+        mock_discover_new_testsets.assert_called_with(
+            db, 'fake-practitest-session')
+        testset_ids = [testset['id'] for testset in TESTSETS]
+        mock_update_testsets.assert_called_with(
+            db, 'fake-practitest-session', testset_ids=testset_ids)
+
+
[e-mail]('qakit.metrics.practitest.instances.update_testsets_instances')
+class UpdateInstancesTestCase(unittest.TestCase):
+
+    def test_testset_ids_specified(self, mock_update_testsets_instances):
+        test_execution.update_instances(
+            'fake-db', 'fake-practitest-session', 'fake-testset-ids')
+        mock_update_testsets_instances.assert_called_with(
+            'fake-db',
+            'fake-practitest-session',
+            testset_ids='fake-testset-ids')
+
+    def test_no_testset_ids_specified(self, mock_update_testsets_instances):
+        db = mock.Mock(
+            testsets=mock.Mock(
+                find=mock.Mock(return_value=TESTSETS)))
+        test_execution.update_instances(db, 'fake-practitest-session', None)
+        testset_ids = [testset['id'] for testset in TESTSETS]
+        mock_update_testsets_instances.assert_called_with(
+            db,
+            'fake-practitest-session',
+            testset_ids=testset_ids)

=== modified file 'qakit/practitest/practitest.py'
--- qakit/practitest/practitest.py	2015-09-03 13:32:10 +0000
+++ qakit/practitest/practitest.py	2015-10-29 22:05:12 +0000
@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import json
 import logging
 import requests
@@ -28,6 +29,35 @@
 PROD_PRACTITEST = 'https://prod.practitest.com/api'
 
 
+def practitest_time_to_datetime(practitest_time):
+    """Convert from PractiTest time to Python datetime.
+
+    PractiTest times look like '13-Apr-2015  22:02'.
+
+    """
+    return datetime.datetime.strptime(
+        practitest_time, '%d-%b-%Y  %H:%M')
+
+
+def append_last_run_datetime(practitest_dict):
+    """Append a Python datetime for 'last_run' to a given PractiTest dict.
+
+    PractiTest gives us its datetimes as strings; translating to a Python
+    datetime makes querying, sorting, etc. much easier.
+
+    :param practitest_dict: a PractiTest JSON dict such as an instance or a
+        testset
+
+    """
+    try:
+        practitest_dict['last_run_datetime'] = practitest_time_to_datetime(
+            practitest_dict['last_run']['value'])
+    except ValueError:
+        # no last_run, no harm done
+        pass
+    return practitest_dict
+
+
 class PractitestSession:
 
     def __init__(self, project_id, api_key, api_secret_key, user_email=None):
@@ -54,12 +84,14 @@
     def _get(self, url, params={}):
         """Get the given url, retrying on failure."""
         params['project_id'] = self.project_id
-        return requests.get(
+        response = requests.get(
             url=url,
             headers=auth.compose_headers(
                 self.api_key,
                 self.api_secret_key),
             data=json.dumps(params))
+        response.raise_for_status()
+        return response
 
 
     def _get_all(self, url, params={}):
@@ -69,6 +101,8 @@
             return int(pagination['total_entities']) - total_read > 0
 
         json_data = []
+        response = None
+        params['page'] = 1
         while True:
             response = self._get(url, params)
             if not response.ok:
@@ -79,6 +113,7 @@
                 params['page'] = str(pagination['page'] + 1)
             else:
                 break
+        response.raise_for_status()
         return json_data
 
 
@@ -159,7 +194,14 @@
 
         """
         url = PROD_PRACTITEST + '/sets/{}.json'.format(id)
-        return self._get(url).json()
+        try:
+            return self._get(url).json()
+        except requests.exceptions.HTTPError as e:
+            if "TestSet was not found" in e.response.text:
+                raise ValueError("TestSet {} not found!".format(id))
+            else:
+                raise
+            
 
     def create_testset(self, name, priority, device, 
                        level, release, build, buildinfo):
@@ -187,7 +229,13 @@
 
         """
         url = PROD_PRACTITEST + '/sets/{}/instances.json'.format(id)
-        return self._get_all(url)
+        try:
+            return self._get_all(url)
+        except requests.exceptions.HTTPError as e:
+            if "TestSet was not found" in e.response.text:
+                raise ValueError("TestSet {} not found!".format(id))
+            else:
+                raise
 
     def create_instances(self, set_id, test_ids):
         """