Merge lp:~fginther/ubuntu-ci-services-itself/add-lp-build-logs into lp:ubuntu-ci-services-itself

[Francis Ginther]

Pass the completed subticket buildlogs to the lander and have it provide subticket status updates.

Pass the completed subticket buildlogs to the lander and have it provide subticket status updates.

The subticket status updates are already being passed back, the just turns on the lander functionality to update the ticket.

Here's a ticket with the subticket artifacts and progress state updates: http://15.125.114.120/ticket.html?ticket_id=6

While testing this, I ran into lp:1291591. I discovered that the check_ppa code which looks for prior uploads was running *after* the upload_source_packages method which does the dput to the ppa. So it's been pure timing luck that we don't see an upload failure on every ticket. I finally hit case where my new source package made it to the ppa before check_ppa executed.

The fix is fairly simply, split upload_packages.upload_source_packages() into two methods, one to crete the upload list, the other to do the dput and then call check_ppa() in between. This was a smaller fix, so I rolled it into here.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:372
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/397/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/397/rebuild

review: Needs Fixing (continuous-integration)

[lp:~fginther/ubuntu-ci-services-itself/add-lp-build-logs]

373. By Francis Ginther on 2014-03-13

Pep8 fix.

[Andy Doan (doanac)]

yeah!

review: Approve

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:373
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/398/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/398/rebuild

review: Approve (continuous-integration)

[Chris Johnston (cjohnston)]

review: Approve

=== modified file 'branch-source-builder/bsbuilder/run_worker.py'
--- branch-source-builder/bsbuilder/run_worker.py	2014-03-12 05:26:00 +0000
+++ branch-source-builder/bsbuilder/run_worker.py	2014-03-13 03:31:31 +0000
@@ -51,9 +51,9 @@
     The return is an artifact entry for the progress message.'''
     package_build_result_filename = 'package_build.output.log'
     body = '{}\n\n'.format(content['message'])
-    body += 'Uploaded source packages:\n'
+    body += 'Uploaded source packages:'
     for subticket_status in content['subticket_status']:
-        body += 'name: {}\n'.format(subticket_status['name'])
+        body += '\nname: {}\n'.format(subticket_status['name'])
         body += 'version: {}\n'.format(subticket_status['version'])
         body += 'status: {}\n'.format(subticket_status['status_text'])
         if subticket_status.get('message'):
@@ -88,7 +88,7 @@
             'artifacts': [],
         }
 
-        upload_list = upload_package.upload_source_packages(ppa, subtickets)
+        upload_list = upload_package.create_upload_list(subtickets)
         log.info('upload_list: {}'.format(upload_list))
 
         if params.get('cancelled'):
@@ -108,6 +108,9 @@
                 create_result_file(datastore, out_data))
             amqp_utils.progress_failed(trigger, out_data)
         else:
+            # After checking the PPA, it's now safe to do the upload
+            upload_package.upload_source_packages(ppa, upload_list)
+
             # Set ret to 1, the upload is failed until it succeeds
             ret = 1
             start_time = time.time()
@@ -117,7 +120,8 @@
                     return amqp_utils.progress_failed, out_data
 
                 (ret, subticket_status) = watch_ppa.watch_ppa(
-                    start_time, series, ppa, archive, None, upload_list)
+                    datastore, start_time, series, ppa, archive, None,
+                    upload_list)
                 log.info('subticket_status: {}'.format(subticket_status))
                 out_data['subticket_status'] = subticket_status
                 if ret == -1:

=== modified file 'branch-source-builder/bsbuilder/upload_package.py'
--- branch-source-builder/bsbuilder/upload_package.py	2014-03-10 22:11:05 +0000
+++ branch-source-builder/bsbuilder/upload_package.py	2014-03-13 03:31:31 +0000
@@ -119,11 +119,16 @@
                             'Not found: {}'.format(package_file))
 
 
-def upload_source_packages(ppa, subtickets):
+def create_upload_list(subtickets):
+    '''Parses the subtickets into the upload list.'''
+    upload_list = get_source_files(subtickets)
+    parse_source_files(upload_list)
+    return upload_list
+
+
+def upload_source_packages(ppa, upload_list):
     '''Attempts source file upload into the PPA.'''
     logging.info('Upload to the ppa: {}'.format(ppa))
-    upload_list = get_source_files(subtickets)
-    parse_source_files(upload_list)
     for package in upload_list:
         packagemanager.upload_package(package['name'], package['version'],
                                       ppa, package['location'])
@@ -133,8 +138,6 @@
             logging.info('Removing temp directory: {}'.format(directory))
             shutil.rmtree(directory)
 
-    return upload_list
-
 
 def main():
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s "

=== modified file 'branch-source-builder/watch_ppa.py'
--- branch-source-builder/watch_ppa.py	2014-03-11 15:43:21 +0000
+++ branch-source-builder/watch_ppa.py	2014-03-13 03:31:31 +0000
@@ -95,7 +95,50 @@
     return launchpadmanager.get_ppa(ppa)
 
 
-def create_package_status(package, step, status, message=None):
+def get_launchpad_log(url):
+    '''Retrieves the build log from launchpad (thanks to celso).'''
+    if not url:
+        return None
+    api_url = url.replace('/launchpad.net/', '/api.launchpad.net/devel/')
+    lp = launchpadmanager.get_launchpad()
+    return lp._browser.get(api_url)
+
+
+def get_build_logs(datastore, ppa, package):
+    '''Constructs artifacts from the build logs for the given package.'''
+    source_name = package.source_name
+    version = package.version
+
+    collection = ppa.getPublishedSources(source_name=source_name,
+                                         version=version)
+    builds = collection[0].getBuilds()
+
+    artifacts = []
+    for build in builds:
+        build_log = get_launchpad_log(build.build_log_url)
+        upload_log = get_launchpad_log(build.upload_log_url)
+        if build_log:
+            artifact_name = 'package_build.{}.{}.build.log'.format(
+                source_name, build.arch_tag)
+            location = datastore.put_file(artifact_name, build_log,
+                                          'text/plain')
+            artifacts.append({
+                'name': artifact_name,
+                'reference': location,
+                'type': 'LOGS'})
+        if upload_log:
+            artifact_name = 'package_build.{}.{}.upload.log'.format(
+                source_name, build.arch_tag)
+            location = datastore.put_file(artifact_name, upload_log,
+                                          'text/plain')
+            artifacts.append({
+                'name': artifact_name,
+                'reference': location,
+                'type': 'LOGS'})
+    return artifacts
+
+
+def create_package_status(package, step, status, message=None, artifacts=None):
     return {
         'name': package.source_name,
         'version': package.version,
@@ -104,11 +147,13 @@
         'status': status.value,
         'status_text': status.title,
         'message': message,
+        'artifacts': artifacts,
     }
 
 
-def collect_subticket_status(ppa, packages_not_in_ppa, packages_building,
-                             packages_failed, packages_complete):
+def collect_subticket_status(datastore, ppa, packages_not_in_ppa,
+                             packages_building, packages_failed,
+                             packages_complete):
     '''Collects the sets of package build status into a subticket_status.'''
     subticket_status = []
     for package in packages_not_in_ppa:
@@ -125,13 +170,15 @@
         subticket_status.append(create_package_status(
             package, SubTicketWorkflowStep.COMPLETED,
             SubTicketWorkflowStepStatus.PKG_BUILDING_FAILED,
-            FAILED_SOURCE_MESSAGE.format(ppa.web_link)))
+            FAILED_SOURCE_MESSAGE.format(ppa.web_link),
+            get_build_logs(datastore, ppa, package)))
 
     for package in packages_complete:
         subticket_status.append(create_package_status(
             package, SubTicketWorkflowStep.COMPLETED,
             SubTicketWorkflowStepStatus.PKG_BUILDING_COMPLETED,
-            PASSED_SOURCE_MESSAGE.format(ppa.web_link)))
+            PASSED_SOURCE_MESSAGE.format(ppa.web_link),
+            get_build_logs(datastore, ppa, package)))
 
     return subticket_status
 
@@ -206,7 +253,7 @@
     return None
 
 
-def watch_ppa(time_start, series, ppa, dest_ppa, arch, upload_list):
+def watch_ppa(datastore, time_start, series, ppa, dest_ppa, arch, upload_list):
     # Prepare launchpad connection:
     lp_series = launchpadmanager.get_series(series)
     monitored_ppa = launchpadmanager.get_ppa(ppa)
@@ -264,9 +311,9 @@
         packages_not_in_ppa, packages_building, packages_failed,
         packages_complete, arch)
 
-    status = collect_subticket_status(monitored_ppa, packages_not_in_ppa,
-                                      packages_building, packages_failed,
-                                      packages_complete)
+    status = collect_subticket_status(datastore, monitored_ppa,
+                                      packages_not_in_ppa, packages_building,
+                                      packages_failed, packages_complete)
     for package in status:
         subticket = find_subticket(upload_list, package['name'],
                                    package['version'])

=== modified file 'lander/bin/lander_service_wrapper.py'
--- lander/bin/lander_service_wrapper.py	2014-03-11 19:33:29 +0000
+++ lander/bin/lander_service_wrapper.py	2014-03-13 03:31:31 +0000
@@ -49,14 +49,18 @@
 class MessageSync(object):
     progress = logging.getLogger('PROGRESS_TRIGGER')
 
-    def __init__(self, ticket, started_cb=None, completed_cb=None):
+    def __init__(self, ticket, started_cb=None, completed_cb=None,
+                 message_cb=None):
         if not started_cb:
             started_cb = lambda: True
         if not completed_cb:
             completed_cb = lambda x: True
+        if not message_cb:
+            message_cb = lambda x: True
         self.ticket = ticket
         self.started_cb = started_cb
         self.completed_cb = completed_cb
+        self.message_cb = message_cb
 
     def on_message(self, msg):
         data = json.loads(msg.body)
@@ -69,6 +73,8 @@
         else:
             self.progress.info(state)
 
+        self.message_cb(data)
+
         if state == 'STATUS':
             self.started_cb()
         elif state == 'COMPLETED':
@@ -86,8 +92,8 @@
 
 
 def _drain_progress(amqp_config, queue, ticket, started_cb=None,
-                    completed_cb=None):
-    sync = MessageSync(ticket, started_cb, completed_cb)
+                    completed_cb=None, message_cb=None):
+    sync = MessageSync(ticket, started_cb, completed_cb, message_cb)
     amqp_utils.process_queue(amqp_config, queue, sync.on_message, delete=True)
     return sync.data
 
@@ -195,7 +201,8 @@
         logger.info('starting progress handler...')
         return _drain_progress(args.amqp_config, queue, ticket,
                                ticket.set_pkg_inprogress,
-                               ticket.set_pkg_complete)
+                               ticket.set_pkg_complete,
+                               ticket.set_subticket_progress)
     except:
         ticket.set_pkg_complete(False)
         raise

=== modified file 'lander/bin/ticket_api.py'
--- lander/bin/ticket_api.py	2014-03-11 03:22:51 +0000
+++ lander/bin/ticket_api.py	2014-03-13 03:31:31 +0000
@@ -69,6 +69,19 @@
             state = TicketWorkflowStepStatus.PKG_BUILDING_FAILED
         self._patch_pkg_status(state)
 
+    def _patch_subticket_status(self, subticket_id, step, status):
+        url = self._url + '/updatesubticketstatus/' + str(subticket_id) + '/'
+        self._patch(url, {'status': status, 'current_workflow_step': step})
+
+    def set_subticket_progress(self, data):
+        '''Processes the content of messages with subticket_status.
+
+        Not all messages will have this content.'''
+        for subticket in data.get('subticket_status', []):
+            self._patch_subticket_status(subticket['id'], subticket['step'],
+                                         subticket['status'])
+            self.process_artifacts(subticket, subticket['id'])
+
     def _patch_ticket_status(self, status):
         step = TicketWorkflowStep.IMAGE_BUILDING.value
         status = status.value
@@ -180,10 +193,12 @@
         multiple updates. Attempts to create duplicate artifacts should be
         caught when the artifact is placed in the data store by the creator.'''
         current_artifacts = self.get_artifact_list(subticket_id)
-        for artifact in data.get('artifacts', []):
-            if artifact['name'] not in current_artifacts:
-                self.add_artifact(artifact['name'], artifact['reference'],
-                                  artifact['type'], subticket_id)
+        artifact_list = data.get('artifacts', None)
+        if artifact_list:
+            for artifact in artifact_list:
+                if artifact['name'] not in current_artifacts:
+                    self.add_artifact(artifact['name'], artifact['reference'],
+                                      artifact['type'], subticket_id)
 
     def get_full_ticket(self):
         url = self._url + '/fullticket/' + self._ticket_id + '/'