Merge lp:~mabac/svammel/re-implement-harvesting-code into lp:svammel

[Mattias Backman]

Hi,

This branch changes the build log harvesting and filtering into a more straight on approach.

Now the script begins with harvesting all builds with 'Failed to build' status. This step also includes collating all builds by source package. Then we filter out packages which are not interesting (those are packages with no fails on armel and those which failed on armel but also failed on another architecture).

Then we go ahead and file bugs for all the packages left after filtering.

Thanks,

Mattias

[James Westby (james-w)]

Hi,

It looks like this contains your cleanup changes as well.

If you want to do this then submitting the merge proposal with
a "pre-requisite branch" will mean that we can review just the
diff of these changes, without seeing the changes from the other
branch again.

You can do this from the "Extra options" expander when submitting.
If you hit "Resubmit proposal" on this one then you should be able
to add the other branch as a pre-requisite to this merge proposal.

Thanks,

James

review: Needs Information

[lp:~mabac/svammel/re-implement-harvesting-code]

40. By Mattias Backman on 2011-03-09

Move magic string into constant.

41. By Mattias Backman on 2011-03-09

Merge trunk.

42. By Mattias Backman on 2011-03-09

Merge trunk.

Unmerged revisions

=== modified file 'bug_reporting.py'
--- bug_reporting.py	2011-03-02 09:18:43 +0000
+++ bug_reporting.py	2011-03-09 08:06:52 +0000
@@ -63,7 +63,4 @@
     # duplicate if the bug is an ftbfs and it is in the arm-porting-queue.
     dupe_bugs = project.searchTasks(tags=dupe_tags, tags_combinator='All')
 
-    # Collection size is not available, so iterate to see if there are items.
-    for bug in dupe_bugs:
-        return True
-    return False
+    return len(dupe_bugs[:1]) > 0

=== modified file 'data_parsing.py'
--- data_parsing.py	2011-03-02 09:18:03 +0000
+++ data_parsing.py	2011-03-09 08:06:52 +0000
@@ -30,8 +30,8 @@
 all_packages = dict()
 all_spph = dict()
 
-all_packages_file = 'tests/serial/all_packages.dump'
-all_spph_file = 'tests/serial/all_spph.dump'
+
+FAILED_BUILD_STATE = 'Failed to build'
 
 
 class InvalidArchiveError(Exception):
@@ -55,57 +55,13 @@
     return tags
 
 
-# Copied from build_status.py
-# Copyright (c) 2007-2010 Michael Bienia <[e-mail]>
-# Modified by Mattias Backman <[e-mail]>
-class SourcePackage(object):
-    class VersionList(list):
-        def append(self, item):
-            super(SourcePackage.VersionList, self).append(item)
-            self.sort(key=lambda x: x.version, cmp=apt_pkg.VersionCompare)
-
-    def __init__(self, srcpkg):
-        self.name = srcpkg.source_package_name
-        self.component = srcpkg.component_name
-        self.url = get_base_url() + '+source/%s' % self.name
-        self.apiurl = get_base_api_url() + '+source/%s' % self.name
-        self.versions = self.VersionList()
-        all_packages[self.name] = self
-
-    def is_ftbfs(self, arch_list, current=True):
-        ''' Returns True if at least one FTBFS exists. '''
-        for ver in self.versions:
-            if ver.current != current:
-                continue
-            for arch in arch_list:
-                log = ver.get_arch(arch)
-                if log and log.buildstate != 'Needs building':
-                    return True
-        return False
-
-    def get_ftbfs_logs(self, arch):
-        ''' Returns logs for failed builds on arch. '''
-        fail_logs = []
-        for ver in self.versions:
-            buildlog = ver.get_arch(arch)
-            if buildlog and buildlog.buildstate == 'Failed to build':
-                fail_logs.append((ver.version, buildlog))
-        return fail_logs
-
-
-# Copied from build_status.py
-# Copyright (c) 2007-2010 Michael Bienia <[e-mail]>
-# Modified by Mattias Backman <[e-mail]>
 class SPPH(object):
     def __init__(self, spph_link):
         self.spph = get_launchpad().load(spph_link)
         self.logs = dict()
         self.version = self.spph.source_package_version
-        self.pocket = self.spph.pocket
-        self.srcpkg = all_packages.get(self.spph.source_package_name)
-        if not self.srcpkg:
-            self.srcpkg = SourcePackage(self.spph)
-        self.srcpkg.versions.append(self)
+        self.package_name = self.spph.source_package_name
+        self.url = get_base_api_url() + '+source/%s' % self.package_name
         self.current = None
 
     def add_build_log(self, buildlog):
@@ -115,19 +71,16 @@
         return self.logs.get(arch)
 
 
-# Copied from build_status.py
-# Copyright (c) 2007-2010 Michael Bienia <[e-mail]>
-# Modified by Mattias Backman <[e-mail]>
 def fetch_pkg_list(archive, series, main_archive=None):
-    state = 'Failed to build'
     print_message('The following packages have failed to build.')
 
     # XXX wgrant 2009-09-19: This is an awful hack. We should really
     # just let IArchive.getBuildRecords take a series argument.
+    # https://bugs.launchpad.net/launchpad/+bug/558907
     if archive.name == 'primary':
-        buildlist = series.getBuildRecords(build_state=state)
+        buildlist = series.getBuildRecords(build_state=FAILED_BUILD_STATE)
     else:
-        buildlist = archive.getBuildRecords(build_state=state)
+        buildlist = archive.getBuildRecords(build_state=FAILED_BUILD_STATE)
 
     for build in buildlist:
         csp_link = build.current_source_publication_link
@@ -135,53 +88,40 @@
             # Build log for an older version
             continue
 
-        print_message("  %s" % build.title)
+        print_message("  Found build '%s'" % build.title)
 
         spph = all_spph.get(csp_link)
         if not spph:
             spph = all_spph[csp_link] = SPPH(csp_link)
 
-        if spph.current is None:
-            # If a main archive is specified, we check if the current source
-            # is still published there. If it isn't, then it's out of date.
-            # We should make this obvious.
-            # The main archive will normally be the primary archive, and
-            # probably only makes sense if the target archive is a rebuild.
-            if main_archive:
-                main_publications = main_archive.getPublishedSources(
-                    distro_series=series,
-                    exact_match=True,
-                    source_name=spph.spph.source_package_name,
-                    version=spph.spph.source_package_version,
-                    status='Published')
-                spph.current = len(main_publications[:1]) > 0
-            else:
-                spph.current = True
-
-        if not spph.current:
-            print_message("    superseded")
         spph.add_build_log(build)
 
-    print_message("Done processing '%s'" % state)
-    return all_packages, all_spph
-
-
-def get_selective_platform_fail(source_package, fail_platforms, ok_platforms):
-    """Determine if the build fail occurs only on certain platforms.
+    return all_spph
+
+
+def filter_spph(all_spph_list, fail_arch, ok_archs):
+    """Filter out the build failures that are unique to the target arch.
 
     Arguments:
-    source_package -- The source package to investigate.
-    fail_platforms -- A list of platforms for which we search for build fails.
-    ok_plaforms -- A list of platforms that should have built OK.
+    all_spph_list -- List of all spphs to investigate.
+    fail_arch -- The arch on which the builds should have failed.
+    ok_archs -- A list of archs on which the builds have not failed.
 
     Returns:
-    True if any of fail_platforms have failed to build and
-    there are no failures in ok_platforms.
+    A subset of all_spph_list which represents the interesting packages.
     """
-    if source_package.is_ftbfs(arch_list=fail_platforms, current=True):
-        return not source_package.is_ftbfs(arch_list=ok_platforms,
-                                          current=True)
-    return False
+    filtered_list = []
+    for spph in all_spph_list.values():
+        if spph.get_arch(fail_arch) is None:
+            # Drop this spph since it did not fail on target arch.
+            continue
+        if next((arch for arch in ok_archs if spph.get_arch(arch)),
+                None) is not None:
+            # Drop this spph since it did fail on other arch.
+            continue
+        filtered_list.append(spph)
+
+    return filtered_list
 
 
 def get_build_status(archive_name, series_name):

=== modified file 'file-failures.py'
--- file-failures.py	2011-03-07 09:57:11 +0000
+++ file-failures.py	2011-03-09 08:06:52 +0000
@@ -13,7 +13,7 @@
 from data_parsing import (
     get_tags_for_fail,
     get_build_status,
-    get_selective_platform_fail,
+    filter_spph,
     )
 from bug_reporting import (
     file_bug_by_url,
@@ -44,12 +44,20 @@
 set_verbose_messages(args.verbose)
 
 init(args.serviceroot, 'testing', '~/.launchpadlib/cache/')
+
+# We are interested in build failures for this platform:
+fail_platform = 'armel'
+# unless there's a failed build for any of these:
+ok_platforms = ['i386', 'amd64']
+
 try:
-    all_packages, all_spph = get_build_status(args.archive, args.series)
+    interesting_spph = filter_spph(get_build_status(args.archive, args.series),
+                                   fail_platform, ok_platforms)
 except:
     print 'Error: Could not get build status'
     raise
 
+<<<<<<< TREE
 # We are interested in build failures for these platforms:
 fail_platforms = ['armel']
 # unless there's a failed build for any of these:
@@ -79,6 +87,27 @@
                     (sp.name, bug_description[0:220], bug_title, bug_tags))
                 file_bug_by_url(sp.apiurl, bug_description, bug_title,
                                 bug_tags, get_launchpad())
+=======
+for spph in interesting_spph:
+
+    log = spph.get_arch(fail_platform)
+    if log is not None:
+        build_log = get_file_contents(log.build_log_url)
+>>>>>>> MERGE-SOURCE
     else:
-        print_message("No reason to file a bug for %s." % sp.name)
-    print_message('\n--')
+        print "%s build log not available for %s version %s." % \
+            (fail_platform, spph.package_name, spph.version)
+        build_log = ''
+    bug_tags = get_tags_for_fail(build_log)
+    interesting_log_part = get_interesting_part(build_log)
+
+    bug_title = '%s version %s failed to build on %s' % \
+        (spph.package_name, spph.version, log.arch_tag)
+    bug_description = '%s\n%s\n\n%s' % \
+        (bug_title, log.web_link, interesting_log_part)
+
+    print_message(
+        "Will file bug for %s\n desc: %s\n title:  %s\n tags: %s" %
+        (spph.package_name, bug_description[0:220], bug_title, bug_tags))
+    file_bug_by_url(spph.url, bug_description, bug_title,
+                    bug_tags, get_launchpad())

=== removed file 'serialization.py'
--- serialization.py	2011-02-28 14:51:09 +0000
+++ serialization.py	1970-01-01 00:00:00 +0000
@@ -1,23 +0,0 @@
-###############################################################################
-# Copyright (c) 2011, 2011 Linaro
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-#
-# Contributors:
-#     Linaro Infrastructure Team - initial implementation
-###############################################################################
-import pickle
-
-
-def dump_object(obj, f):
-    """Dump obj to f. """
-    with open(f, 'wb') as pickle_file:
-        pickle.dump(obj, pickle_file)
-
-
-def load_object(f):
-    """Load from f and return the object."""
-    with open(f, 'rb') as from_file:
-        return pickle.load(from_file)

=== removed file 'serialize-packages.py'
--- serialize-packages.py	2011-02-28 14:51:09 +0000
+++ serialize-packages.py	1970-01-01 00:00:00 +0000
@@ -1,38 +0,0 @@
-###############################################################################
-# Copyright (c) 2011, 2011 Linaro
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Eclipse Public License v1.0
-# which accompanies this distribution, and is available at
-# http://www.eclipse.org/legal/epl-v10.html
-#
-# Contributors:
-#     Linaro Infrastructure Team - initial implementation
-###############################################################################
-from serialization import (
-    dump_object,
-    )
-from data_parsing import (
-    get_build_status,
-    all_packages,
-    all_spph,
-    all_packages_file,
-    all_spph_file,
-    )
-
-
-def dump_build_status():
-    dump_object(all_spph, all_spph_file)
-    dump_object(all_packages, all_packages_file)
-
-    print """
-              Done serializing build data:
-                  all_packages -> %s
-                  all_spph -> %s
-          """ % (all_packages_file, all_spph_file)
-
-
-print """Downloading build data"""
-get_build_status()
-
-print """Serializing objects"""
-dump_build_status()

=== removed directory 'tests/serial'