Merge lp:~fginther/ubuntu-ci-services-itself/bsbuilder-fixes into lp:ubuntu-ci-services-itself

[Francis Ginther]

Pass subtickets directly to the branch source builder, allowing it to create subticket progress updates that are then updated by the lander. Check contents of the PPAs before uploading to ensure that the build will succeed. Create a results file to pass back as a global result artifact and pass a failure status when the build fails.

Pass subtickets directly to the branch source builder, allowing it to create subticket progress updates that are then updated by the lander. Check contents of the PPAs before uploading to ensure that the build will succeed. Create a results file to pass back as a global result artifact and pass a failure status when the build fails.

[lp:~fginther/ubuntu-ci-services-itself/bsbuilder-fixes]

335. By Francis Ginther on 2014-03-10

Merge to trunk.

336. By Francis Ginther on 2014-03-10

Merge trunk again.

[Francis Ginther (fginther)]

Found 2 issues in the result file:
1) mime-type isn't being set.
2) \n, not /n, when joining messages.

review: Needs Fixing

[lp:~fginther/ubuntu-ci-services-itself/bsbuilder-fixes]

337. By Francis Ginther on 2014-03-10

Fix content_type and us '\n' instead of '/n' when joining messages.

[Andy Doan (doanac)]

this was broken up and merged.

Unmerged revisions

337. By Francis Ginther on 2014-03-10

Fix content_type and us '\n' instead of '/n' when joining messages.

336. By Francis Ginther on 2014-03-10

Merge trunk again.

335. By Francis Ginther on 2014-03-10

Merge to trunk.

334. By Francis Ginther on 2014-03-10

More renaming and a few renaming fixes.

333. By Francis Ginther on 2014-03-10

Remove _get_versions_for_source_package_in_ppa added to cupstream2distro.

332. By Francis Ginther on 2014-03-09

Some minor refactoring and renaming.

331. By Francis Ginther on 2014-03-08

Updates from first round of deployed testing.

330. By Francis Ginther on 2014-03-07

Pass subtickets directly to the branch source builder, allowing it to create subticket progress updates that are then updated by the lander. Check contents of the PPAs before uploading to ensure that the build will succeed. Create a results file to pass back as a global result artifact.

=== modified file 'branch-source-builder/bsbuilder/resources/v1.py'
--- branch-source-builder/bsbuilder/resources/v1.py	2014-03-10 01:35:59 +0000
+++ branch-source-builder/bsbuilder/resources/v1.py	2014-03-10 20:31:16 +0000
@@ -31,11 +31,11 @@
     return restish_utils.json_ok(status.results)
 
 
-def _build_source(ticket_id, source_packages, series, ppa, archive,
+def _build_source(ticket_id, subtickets, series, ppa, archive,
                   progress_trigger):
     params = {
         'ticket_id': ticket_id,
-        'source_packages': source_packages,
+        'subtickets': subtickets,
         'series': series,
         'ppa': ppa,
         'archive': archive,
 
=== modified file 'branch-source-builder/bsbuilder/run_worker.py'
--- branch-source-builder/bsbuilder/run_worker.py	2014-03-06 05:57:13 +0000
+++ branch-source-builder/bsbuilder/run_worker.py	2014-03-10 20:31:16 +0000
@@ -16,12 +16,18 @@
 
 import json
 import logging
+import os
 import time
+import yaml
 
 import upload_package
 import watch_ppa
 
-from ci_utils import amqp_utils, dump_stack
+from ci_utils import (
+    amqp_utils,
+    data_store,
+    dump_stack,
+)
 
 logging.basicConfig(level=logging.INFO)
 log = logging.getLogger(__name__)
@@ -29,51 +35,128 @@
 TIME_BETWEEN_CHECKS = 60
 
 
+def get_auth_config():
+    DEFAULT_CONFIG_YAML = os.path.join(
+        os.path.dirname(__file__), '../../unit_config')
+    with open(DEFAULT_CONFIG_YAML) as f:
+        config = yaml.safe_load(f)
+    return config
+
+
+def create_result_file(datastore, content):
+    '''Creates a result file based on the final upload and build status.
+
+    The input content has the following format:
+        {
+            'message': 'The high level status of the upload',
+            'subticket_status': [{
+                'name': source_package_name,
+                'version': source_package_version,
+                'step': numeric_workflow_step,
+                'step_text': text_workflow_step,
+                'status': numeric_workflow_status,
+                'status_text': text_workflow_status,
+                'message': subticket_specific_message,
+            }]
+        }
+
+    The return is an artifact entry for the progress message.'''
+    package_build_result_filename = 'package_build.output.log'
+    body = '{}\n\n'.format(content['message'])
+    body += 'Uploaded source packages:\n'
+    for subticket_status in content['subticket_status']:
+        body += 'name: {}\n'.format(subticket_status['name'])
+        body += 'version: {}\n'.format(subticket_status['version'])
+        body += 'status: {}\n'.format(subticket_status['status_text'])
+        if subticket_status.get('message'):
+            body += 'message: {}\n'.format(subticket_status['message'])
+    location = datastore.put_file(package_build_result_filename, str(body),
+                                  content_type='text/plain')
+    return {
+        'name': package_build_result_filename,
+        'type': 'LOGS',
+        'reference': location,
+    }
+
+
 def on_message(msg):
     log.info('on_message: {}'.format(msg.body))
     params = json.loads(msg.body)
-    sources = params['source_packages']
+    ticket_id = params['ticket_id']
+    subtickets = params['subtickets']
     series = params['series']
     ppa = params['ppa']
     archive = params['archive']
-    log.info('The PPA is: {}'.format(ppa))
     trigger = params['progress_trigger']
 
     # Setup the output data to send back to the caller
-    # TODO: sources_packages are just the passed in source files, this
-    #       can be made to be more useful.
     # TODO: artifacts will be populated with artifacts from this build.
-    out_data = {'source_packages': sources,
-                'ppa': ppa,
-                'artifacts': []}
+    out_data = {
+        'message': 'Build in progress.',
+        'ppa': ppa,
+        'archive': archive,
+        'artifacts': [],
+    }
     amqp_utils.progress_update(trigger, params)
 
     try:
-        upload_list = upload_package.upload_source_packages(ppa, sources)
+        datastore = data_store.create_for_ticket(ticket_id, get_auth_config())
+
+        upload_list = upload_package.upload_source_packages(ppa, subtickets)
         log.info('upload_list: {}'.format(upload_list))
-        start_time = time.time()
-        while True:
-            (ret, status) = watch_ppa.watch_ppa(start_time, series, ppa,
-                                                archive, None,
-                                                upload_list)
-            progress = {}
-            for key in status:
-                progress[key] = str(status[key])
-            log.info('progress: {}'.format(progress))
-            out_data['status'] = progress
-            amqp_utils.progress_update(trigger, out_data)
-            if ret == -1:
-                log.info('Going to sleep for {}'.format(TIME_BETWEEN_CHECKS))
-                time.sleep(TIME_BETWEEN_CHECKS)
+
+        # Check that an upload isn't going to fail due to version
+        (failed, subticket_status) = watch_ppa.check_ppa(
+            series, ppa, archive, None, upload_list)
+        if failed:
+            # This is the end of the line, the upload is blocked.
+            out_data['subticket_status'] = subticket_status
+            message = 'Source package upload failed.'
+            log.error(message)
+            out_data['message'] = message
+            out_data['artifacts'].append(
+                create_result_file(datastore, out_data))
+            amqp_utils.progress_failed(trigger, out_data)
+        else:
+            # Set ret to 1, the upload is failed until it succeeds
+            ret = 1
+            start_time = time.time()
+            while True:
+                (ret, subticket_status) = watch_ppa.watch_ppa(
+                    start_time, series, ppa, archive, None, upload_list)
+                log.info('subticket_status: {}'.format(subticket_status))
+                out_data['subticket_status'] = subticket_status
+                if ret == -1:
+                    # -1 indicates that the task is not finished
+                    # Send regular progress back to the lander
+                    amqp_utils.progress_update(trigger, out_data)
+                    log.info('Going to sleep for '
+                             '{}'.format(TIME_BETWEEN_CHECKS))
+                    time.sleep(TIME_BETWEEN_CHECKS)
+                else:
+                    log.info('All done')
+                    break
+
+            if ret == 1:
+                # '1' indicates a failure occurred during the upload/build
+                message = 'A package failed to upload or build.'
+                log.error(message)
+                out_data['message'] = message
+                out_data['artifacts'].append(
+                    create_result_file(datastore, out_data))
+                amqp_utils.progress_failed(trigger, out_data)
             else:
-                log.info('All done')
-                break
-
-        amqp_utils.progress_completed(trigger, out_data)
+                # '0' indicates the build was successful
+                message = 'The package build was successful.'
+                log.error(message)
+                out_data['message'] = message
+                out_data['artifacts'].append(
+                    create_result_file(datastore, out_data))
+                amqp_utils.progress_completed(trigger, out_data)
     except (KeyboardInterrupt, Exception) as e:
-        error_msg = 'Exception: {}'.format(e)
-        log.error(error_msg)
-        out_data['error_message'] = error_msg
+        message = 'Exception: {}'.format(e)
+        log.error(message)
+        out_data['message'] = message
         amqp_utils.progress_failed(trigger, out_data)
         if isinstance(e, KeyboardInterrupt):
             raise  # re-raise so amqp_utils.process_queue can exit
 
=== added file 'branch-source-builder/bsbuilder/tests/test_upload.py'
--- branch-source-builder/bsbuilder/tests/test_upload.py	1970-01-01 00:00:00 +0000
+++ branch-source-builder/bsbuilder/tests/test_upload.py	2014-03-10 20:31:16 +0000
@@ -0,0 +1,89 @@
+# Ubuntu CI Engine
+# Copyright 2013 Canonical Ltd.
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License version 3, as
+# published by the Free Software Foundation.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import mock
+import unittest
+
+import upload_package
+
+
+class TestUploadPackage(unittest.TestCase):
+    '''Tests for the upload_package module.'''
+
+    subticket_list = [{
+        'status': 'New',
+        'current_workflow_step': 'New',
+        'artifact': [{
+            'id': 6,
+            'type': 'SPU',
+            'name': 'package_0.1_source.changes',
+            'reference': 'https://swift.url/package_0.1_source.changes',
+            'resource_uri': '/api/v1/fullsubticketartifact/6/'
+        }, {
+            'id': 5,
+            'type': 'SPU',
+            'name': 'package_0.1.tar.gz',
+            'reference': 'https://swift.url/package_0.1.tar.gz',
+            'resource_uri': '/api/v1/fullsubticketartifact/5/'
+        }, {
+            'id': 4,
+            'type': 'SPU',
+            'name': 'package_0.1.dsc',
+            'reference': 'https://swift.url/package_0.1.dsc',
+            'resource_uri': '/api/v1/fullsubticketartifact/4/'
+        }],
+        'assignee': 'user_id',
+        'source_package_upload': {
+            'version': '0.1',
+            'sourcepackage': {
+                'id': 1,
+                'name': 'package',
+                'resource_uri': '/api/v1/sourcepackage/1/'
+            },
+            'id': 2,
+            'resource_uri': '/api/v1/spu/2/'
+        },
+        'id': 2,
+        'resource_uri': '/api/v1/fullsubticket/2/'
+    }]
+
+    def setUp(self):
+        super(TestUploadPackage, self).setUp()
+
+    def _get_context_manager_mock(self):
+        file_mock = mock.Mock()
+        context_manager_mock = mock.Mock()
+        context_manager_mock.__enter__ = mock.Mock()
+        context_manager_mock.__enter__.return_value = file_mock
+        context_manager_mock.__exit__ = mock.Mock()
+        return context_manager_mock
+
+    @mock.patch('upload_package.create_temp_dir')
+    @mock.patch('__builtin__.open')
+    def test_get_source_files(self, open_mock, temp_dir_mock):
+        temp_dir_mock.return_value = 'mock_location'
+        open_mock.return_value = self._get_context_manager_mock()
+        file_list = []
+        for item in self.subticket_list[0]['artifact']:
+            file_list.append(item['name'])
+
+        result = upload_package.get_source_files(self.subticket_list)
+        self.assertEqual(1, len(result))
+        spu = self.subticket_list[0]['source_package_upload']
+        self.assertEqual(spu['sourcepackage']['name'], result[0]['name'])
+        self.assertEqual(spu['version'], result[0]['version'])
+        self.assertEqual('/api/v1/fullsubticket/2/', result[0]['resource'])
+        self.assertEqual('mock_location', result[0]['location'])
+        self.assertItemsEqual(file_list, result[0]['files'])
 
=== modified file 'branch-source-builder/bsbuilder/tests/test_v1.py'
--- branch-source-builder/bsbuilder/tests/test_v1.py	2014-03-10 01:35:59 +0000
+++ branch-source-builder/bsbuilder/tests/test_v1.py	2014-03-10 20:31:16 +0000
@@ -55,7 +55,7 @@
         get_config.return_value = None
         params = {
             'ticket_id': 1,
-            'source_packages': 'foo',
+            'subtickets': 'foo',
             'series': 'foo',
             'ppa': 'foo',
             'archive': 'foo',
@@ -74,7 +74,7 @@
         send.return_value = None
         params = {
             'ticket_id': 1,
-            'source_packages': 'foo',
+            'subtickets': 'foo',
             'series': 'foo',
             'ppa': 'foo',
             'archive': 'foo',
 
=== modified file 'branch-source-builder/cupstream2distro/packageinppamanager.py'
--- branch-source-builder/cupstream2distro/packageinppamanager.py	2014-02-19 16:34:46 +0000
+++ branch-source-builder/cupstream2distro/packageinppamanager.py	2014-03-10 20:31:16 +0000
@@ -49,12 +49,6 @@
             rev = _get_current_rev_from_config(substract[0])
             branch = _get_parent_branch(substract[0])
             result.add((substract[0], version, rev, branch))
-    foo = set()
-    foo.add(('autopilot', '1.3.1+13.10.20131003.1-0ubuntu2~fginther.1',
-            '100', 'lp:autopilot'))
-    foo.add(('autopilot', '1.3.1+13.10.20131003.1-0ubuntu2~fginther.2',
-            '105', 'lp:autopilot'))
-    return foo
     return result
 
 
@@ -74,7 +68,9 @@
     return result
 
 
-def update_all_packages_status(packages_not_in_ppa, packages_building, packages_failed, particular_arch=None):
+def update_all_packages_status(packages_not_in_ppa, packages_building,
+                               packages_failed, packages_complete,
+                               particular_arch=None):
     '''Update all packages status, checking in the ppa'''
 
     for current_package in (packages_not_in_ppa.union(packages_building)):
@@ -93,6 +89,7 @@
             elif package_status == PackageInPPA.PUBLISHED:
                 _ensure_removed_from_set(packages_building, current_package)  # in case we missed the "build" step
                 _ensure_removed_from_set(packages_not_in_ppa, current_package)  # in case we missed the "wait" step
+                packages_complete.add(current_package)
 
 
 def _get_current_packaging_version_from_config(source_package_name):
 
=== modified file 'branch-source-builder/setup.py'
--- branch-source-builder/setup.py	2014-02-28 09:25:47 +0000
+++ branch-source-builder/setup.py	2014-03-10 20:31:16 +0000
@@ -25,6 +25,10 @@
 requires = [
     'WebTest==2.0.10',
     'amqplib==1.0.0',
+    'chardet>=2.0.1',
+    'dput>=1.6',
+    'launchpadlib==1.10.2',
+    'lazr.enum>=1.1.2',
     'mock==1.0.1',
     'restish==0.12.1',
 ]
 
=== modified file 'branch-source-builder/upload_package.py'
--- branch-source-builder/upload_package.py	2014-02-19 16:34:46 +0000
+++ branch-source-builder/upload_package.py	2014-03-10 20:31:16 +0000
@@ -64,18 +64,28 @@
         raise EnvironmentError('Failed to open url [{}]: {}'.format(url, e))
 
 
-def get_source_files(source_files):
-    location = create_temp_dir()
-    local_files = []
-    for source in source_files:
-        # Co-locate all the files into a temp directory
-        file_name = os.path.basename(source)
-        file_path = os.path.join(location, file_name)
-        logging.info('Retrieving source file: {}'.format(file_name))
-        with open(file_path, 'w') as out_file:
-            out_file.write(get_url_contents(source))
-        local_files.append(file_name)
-    return (location, local_files)
+def get_source_files(subtickets):
+    '''Retrieves the source package files and creates initial upload_list.'''
+    upload_list = []
+    for subticket in subtickets:
+        package = {}
+        spu = subticket['source_package_upload']
+        package['id'] = spu['id']
+        package['name'] = spu['sourcepackage']['name']
+        package['version'] = spu['version']
+        package['resource'] = subticket['resource_uri']
+        package['location'] = create_temp_dir()
+        package['files'] = []
+        for artifact in subticket['artifact']:
+            source = artifact['reference']
+            file_name = os.path.basename(source)
+            file_path = os.path.join(package['location'], file_name)
+            logging.info('Retrieving source file: {}'.format(file_name))
+            with open(file_path, 'w') as out_file:
+                out_file.write(get_url_contents(source))
+            package['files'].append(file_name)
+        upload_list.append(package)
+    return upload_list
 
 
 def parse_dsc_file(dsc_file):
@@ -87,41 +97,43 @@
                 return arch_lists[0]
 
 
-def parse_source_files(source_directory, source_files):
-    package_list = []
-    for source in source_files:
-        source_name = os.path.join(source_directory, source)
-        if source.endswith('changes'):
+def parse_source_files(upload_list):
+    '''Perform a second pass of the upload list and parse out more info.
+
+    The purpose of this is to make sure all source package files have
+    been downloaded and identify the source package architecture.
+    This must be done after all files in a subticket have been acquired.'''
+    for package in upload_list:
+        for source in package['files']:
+            source_name = os.path.join(package['location'], source)
+            if source.endswith('changes'):
                 changes = parse_changes_file(filename=source_name,
-                                             directory=source_directory)
-                package = {}
-                package['name'] = changes.get('Source')
-                package['version'] = changes.get('Version')
-                package['files'] = []
-                package['files'].append(changes.get_changes_file())
+                                             directory=package['location'])
                 for package_file in changes.get_files():
                     if os.path.exists(package_file):
                         if package_file.endswith('.dsc'):
                             package['architecture'] = parse_dsc_file(
                                 package_file)
-                        package['files'].append(package_file)
                     else:
                         raise EnvironmentError(
                             'Not found: {}'.format(package_file))
-                package_list.append(package)
-    return package_list
-
-
-def upload_source_packages(ppa, upload_files):
+
+
+def upload_source_packages(ppa, subtickets):
     '''Attempts source file upload into the PPA.'''
     logging.info('Upload to the ppa: {}'.format(ppa))
-    (source_directory, source_files) = get_source_files(upload_files)
-    source_packages = parse_source_files(source_directory, source_files)
-    for package in source_packages:
+    upload_list = get_source_files(subtickets)
+    parse_source_files(upload_list)
+    for package in upload_list:
         packagemanager.upload_package(package['name'], package['version'],
-                                      ppa, source_directory)
-    # TODO Return the data set of packages uploaded
-    return source_packages
+                                      ppa, package['location'])
+
+        directory = package['location']
+        if os.path.exists(directory) and os.path.isdir(directory):
+            logging.info('Removing temp directory: {}'.format(directory))
+            shutil.rmtree(directory)
+
+    return upload_list
 
 
 def main():
 
=== modified file 'branch-source-builder/watch_ppa.py'
--- branch-source-builder/watch_ppa.py	2014-02-28 09:25:47 +0000
+++ branch-source-builder/watch_ppa.py	2014-03-10 20:31:16 +0000
@@ -22,7 +22,9 @@
 import json
 import logging
 import os
+import subprocess
 import sys
+import textwrap
 import time
 
 from cupstream2distro import (launchpadmanager, packageinppamanager)
@@ -33,6 +35,27 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../ci-utils'))
 from ci_utils import dump_stack
+from ci_utils.ticket_states import (
+    SubTicketWorkflowStep,
+    SubTicketWorkflowStepStatus
+)
+
+EXISTING_SOURCE_MESSAGE = textwrap.dedent('''\
+        ERROR: A source package with an equal or higher version
+        of {} exists in {}.
+        The upload of {} with version {} is failed.
+        Source package uploads must have a higher version.''')
+FAILED_SOURCE_MESSAGE = textwrap.dedent('''\
+        The source package build has failed in {}.
+        Please review the build logs for the failed package(s).''')
+MISSING_SOURCE_MESSAGE = textwrap.dedent('''\
+        The source package for {} with version {}
+        failed to upload to {}.
+        Please check that the source package is signed with a signature of
+        a launchpad user with permission to upload to the PPA.
+        Also check for a rejection message sent to the package signer.''')
+PASSED_SOURCE_MESSAGE = textwrap.dedent('''\
+        The source packages have been built in {}.''')
 
 
 def parse_arguments():
@@ -72,6 +95,117 @@
     return launchpadmanager.get_ppa(ppa)
 
 
+def create_package_status(package, step, status, message=None):
+    return {
+        'name': package.source_name,
+        'version': package.version,
+        'step': step.value,
+        'step_text': step.title,
+        'status': status.value,
+        'status_text': status.title,
+        'message': message,
+    }
+
+
+def collect_subticket_status(ppa, packages_not_in_ppa, packages_building,
+                             packages_failed, packages_complete):
+    '''Collects the sets of package build status into a subticket_status.'''
+    subticket_status = []
+    for package in packages_not_in_ppa:
+        subticket_status.append(create_package_status(
+            package, SubTicketWorkflowStep.QUEUED,
+            SubTicketWorkflowStepStatus.PKG_BUILDING_WAITING))
+
+    for package in packages_building:
+        subticket_status.append(create_package_status(
+            package, SubTicketWorkflowStep.PKG_BUILDING,
+            SubTicketWorkflowStepStatus.PKG_BUILDING_INPROGRESS))
+
+    for package in packages_failed:
+        subticket_status.append(create_package_status(
+            package, SubTicketWorkflowStep.COMPLETED,
+            SubTicketWorkflowStepStatus.PKG_BUILDING_FAILED,
+            FAILED_SOURCE_MESSAGE.format(ppa.web_link)))
+
+    for package in packages_complete:
+        subticket_status.append(create_package_status(
+            package, SubTicketWorkflowStep.COMPLETED,
+            SubTicketWorkflowStepStatus.PKG_BUILDING_COMPLETED,
+            PASSED_SOURCE_MESSAGE.format(ppa.web_link)))
+
+    return subticket_status
+
+
+def get_versions_for_source_package(series, ppa, source_name):
+    try:
+        source = ppa.getPublishedSources(exact_match=True,
+                                         source_name=source_name,
+                                         distro_series=series)[0]
+        return source.source_package_version
+    except (KeyError, IndexError):
+        return None
+
+
+def check_ppa(series, ppa, dest_ppa, arch, upload_list):
+    # Prepare launchpad connection:
+    lp_series = launchpadmanager.get_series(series)
+    monitored_ppa = launchpadmanager.get_ppa(ppa)
+    if dest_ppa:
+        dest_archive = get_ppa(dest_ppa)
+    else:
+        dest_archive = launchpadmanager.get_ubuntu_archive()
+    logging.info('Series: {}'.format(lp_series))
+    logging.info('Monitoring PPA: {}'.format(monitored_ppa))
+    logging.info('Destination Archive: {}'.format(dest_archive))
+
+    failed = False
+    check_status = []
+    for source_package in upload_list:
+        source = source_package['name']
+        version = source_package['version']
+        logging.info('Inspecting upload: {} - {}'.format(source, version))
+        message_list = []
+        subticket_status = {
+            'name': source,
+            'id': source_package['id'],
+            'version': version,
+        }
+        wf_step = SubTicketWorkflowStep.NEW
+        wf_status = SubTicketWorkflowStepStatus.NEW
+        check_status.append(subticket_status)
+        for ppa in [monitored_ppa, dest_archive]:
+            last_version = get_versions_for_source_package(lp_series, ppa,
+                                                           source)
+            logging.info('Last source: {}'.format(last_version))
+
+            if last_version:
+                try:
+                    subprocess.check_call(['dpkg', '--compare-versions',
+                                           last_version, 'lt', version])
+                except subprocess.CalledProcessError:
+                    # The version in the PPA is equal or higher then the
+                    # source package, the build will fail
+                    wf_step = SubTicketWorkflowStep.COMPLETED
+                    wf_status = SubTicketWorkflowStepStatus.PKG_BUILDING_FAILED
+                    message_list.append(EXISTING_SOURCE_MESSAGE.format(
+                        last_version, ppa.web_link, source, version))
+                    failed = True
+        subticket_status['step'] = wf_step.value
+        subticket_status['step_text'] = wf_step.title
+        subticket_status['status'] = wf_status.value
+        subticket_status['status_text'] = wf_status.title
+        subticket_status['message'] = '\n'.join(message_list)
+
+    return (failed, check_status)
+
+
+def find_subticket(upload_list, name, version):
+    for subticket in upload_list:
+        if subticket['name'] == name and subticket['version'] == version:
+            return subticket
+    return None
+
+
 def watch_ppa(time_start, series, ppa, dest_ppa, arch, upload_list):
     # Prepare launchpad connection:
     lp_series = launchpadmanager.get_series(series)
@@ -82,7 +216,6 @@
         dest_archive = launchpadmanager.get_ubuntu_archive()
     logging.info('Series: {}'.format(lp_series))
     logging.info('Monitoring PPA: {}'.format(monitored_ppa))
-
     logging.info('Destination Archive: {}'.format(dest_archive))
 
     # Get archs available and archs to ignore
@@ -102,6 +235,7 @@
     packages_not_in_ppa = set()
     packages_building = set()
     packages_failed = set()
+    packages_complete = set()
     for source_package in upload_list:
         source = source_package['name']
         version = source_package['version']
@@ -119,8 +253,6 @@
     # to eventually appear in the ppa.
     logging.info('Packages not in PPA: {}'.format(
         list_packages_info_in_str(packages_not_in_ppa)))
-    logging.info('Packages building: {}'.format(packages_building))
-    logging.info('Packages failed: {}'.format(packages_failed))
 
     # Check the status regularly on all packages
     # TODO The following is the original check loop. This can be extracted
@@ -129,11 +261,24 @@
         list_packages_info_in_str(
             packages_not_in_ppa.union(packages_building))))
     packageinppamanager.update_all_packages_status(
-        packages_not_in_ppa, packages_building, packages_failed, arch)
-
-    status = {'pending': packages_not_in_ppa,
-              'building': packages_building,
-              'failed': packages_failed}
+        packages_not_in_ppa, packages_building, packages_failed,
+        packages_complete, arch)
+
+    status = collect_subticket_status(monitored_ppa, packages_not_in_ppa,
+                                      packages_building, packages_failed,
+                                      packages_complete)
+    for package in status:
+        subticket = find_subticket(upload_list, package['name'],
+                                   package['version'])
+        if not subticket:
+            # No match, this should not happen
+            return (1, status)
+
+        # Populate the remaining fields necessary to update ticket status
+        package['id'] = subticket['id']
+        package['resource'] = subticket['resource']
+
+    logging.info("Status: {}".format(status))
     # if we have no package building or failing and have wait for
     # long enough to have some package appearing in the ppa, exit
     if (packages_not_in_ppa and not packages_building and
@@ -143,6 +288,16 @@
         logging.info(
             "Some source packages were never published in the ppa: "
             "{}".format(list_packages_info_in_str(packages_not_in_ppa)))
+        for package in status:
+            # If any packages are WAITING, set them to FAILED
+            if package['status'] == SubTicketWorkflowStepStatus.WAITING.value:
+                package['step'] = SubTicketWorkflowStep.COMPLETED.value,
+                package['status'] = \
+                    SubTicketWorkflowStepStatus.PKG_BUILDING_COMPLETED.value
+            # Add a message indicating the cause of the failure
+            message_list.append(MISSING_SOURCE_MESSAGE.format(
+                package['name'], package['version'], monitored_ppa.web_link))
+        status['message'] = '\n'.join(message_list)
         return (1, status)
 
     # break out of status check loop if all packages have arrived in
 
=== modified file 'juju-deployer/branch-source-builder.yaml.tmpl'
--- juju-deployer/branch-source-builder.yaml.tmpl	2014-03-10 01:35:59 +0000
+++ juju-deployer/branch-source-builder.yaml.tmpl	2014-03-10 20:31:16 +0000
@@ -29,7 +29,7 @@
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
                 unit-config: include-base64://configs/unit_config.yaml
-                packages: "dput python-dput python-swiftclient"
+                packages: "dput python-dput python-swiftclient lazr.enum"
                 install_sources: |
                     - ${CI_PPA}
                 install_keys: |
 
=== modified file 'lander/bin/lander_service_wrapper.py'
--- lander/bin/lander_service_wrapper.py	2014-03-10 14:57:34 +0000
+++ lander/bin/lander_service_wrapper.py	2014-03-10 20:31:16 +0000
@@ -48,14 +48,18 @@
 class MessageSync(object):
     progress = logging.getLogger('PROGRESS_TRIGGER')
 
-    def __init__(self, ticket, started_cb=None, completed_cb=None):
+    def __init__(self, ticket, started_cb=None, completed_cb=None,
+                 message_cb=None):
         if not started_cb:
             started_cb = lambda: True
         if not completed_cb:
             completed_cb = lambda x: True
+        if not message_cb:
+            message_cb = lambda x: True
         self.ticket = ticket
         self.started_cb = started_cb
         self.completed_cb = completed_cb
+        self.message_cb = message_cb
 
     def on_message(self, msg):
         data = json.loads(msg.body)
@@ -68,6 +72,8 @@
         else:
             self.progress.info(state)
 
+        self.message_cb(data)
+
         if state == 'STATUS':
             self.started_cb()
         elif state == 'COMPLETED':
@@ -85,8 +91,8 @@
 
 
 def _drain_progress(amqp_config, queue, ticket, started_cb=None,
-                    completed_cb=None):
-    sync = MessageSync(ticket, started_cb, completed_cb)
+                    completed_cb=None, message_cb=None):
+    sync = MessageSync(ticket, started_cb, completed_cb, message_cb)
     amqp_utils.process_queue(amqp_config, queue, sync.on_message, delete=True)
     return sync.data
 
@@ -169,17 +175,10 @@
 
     queue = '%s-bsbuilder' % config['master']['progress_trigger']
 
-    # Extract the source files from the master request parameters
     request_parameters = ticket.get_full_ticket()
-    source_files = []
-    for subticket in request_parameters.get('subticket', []):
-        for artifact in subticket.get('artifact', []):
-            if artifact.get('type', None) == 'SPU':
-                source_files.append(artifact['reference'])
-
     params = {
         'ticket_id': config['master']['request_id'],
-        'source_packages': source_files,
+        'subtickets': request_parameters['subticket'],
         'series': request_parameters.get('series', ''),
         'ppa': config['ppa_assigner']['ppa'],
         'archive': request_parameters.get('master_ppa', ''),
@@ -193,7 +192,8 @@
         logger.info('starting progress handler...')
         return _drain_progress(args.amqp_config, queue, ticket,
                                ticket.set_pkg_inprogress,
-                               ticket.set_pkg_complete)
+                               ticket.set_pkg_complete,
+                               ticket.set_subticket_progress)
     except:
         ticket.set_pkg_complete(False)
         raise
 
=== modified file 'lander/bin/ticket_api.py'
--- lander/bin/ticket_api.py	2014-03-07 15:31:20 +0000
+++ lander/bin/ticket_api.py	2014-03-10 20:31:16 +0000
@@ -65,6 +65,18 @@
             state = TicketWorkflowStepStatus.PKG_BUILDING_FAILED
         self._patch_pkg_status(state)
 
+    def _patch_subticket_status(self, subticket_id, step, status):
+        url = self._url + '/updatesubticketstatus/' + str(subticket_id) + '/'
+        self._patch(url, {'status': status, 'current_workflow_step': step})
+
+    def set_subticket_progress(self, data):
+        '''Processes the content of messages with subticket_status.
+
+        Not all messages will have this content.'''
+        for subticket in data.get('subticket_status', []):
+            self._patch_subticket_status(subticket['id'], subticket['step'],
+                                         subticket['status'])
+
     def _patch_ticket_status(self, status):
         step = TicketWorkflowStep.IMAGE_BUILDING.value
         status = status.value