Merge lp:~stevenk/launchpad/gina-populate-changelog into lp:launchpad

[Steve Kowalik]

[r=gmb][ui=none][bug=680382] Gina now imports changelogs into the Librarian when processing new source packages.

This branch changes gina (the script that imports sources from Debian into LP) to also upload full changelogs to the librarian, and link the SourcePackageRelease to the changelog.

[Graham Binns (gmb)]

> 8 +Check that the changelog was uploaded to the librarian correctly:
> 9 +
> 10 + >>> print x11p.changelog.content.sha1
> 11 + 759bd2df0d886f9f06c383048afb90eaa81418f6
> 12 +

This looks like either a) sampledata or b) a magic number to me. I can't
see where it came from, and grepping isn't turning anything up. Given
that the changelog is a LibraryFileAlias I'm guessing this is buried in
test data somewhere.

Can you change this so that the sha1 is something obviously
manufactured (e.g. 4242424242424242424242424242424242424242 or
similar)? At the very least, you need to add a comment explaining where
the data comes from in the first place.

review: Needs Fixing (code)

[Graham Binns (gmb)]

review: Approve (code)

=== modified file 'lib/lp/soyuz/doc/gina.txt'
--- lib/lp/soyuz/doc/gina.txt	2010-11-11 11:55:53 +0000
+++ lib/lp/soyuz/doc/gina.txt	2010-11-23 10:58:17 +0000
@@ -215,6 +215,21 @@
     >>> x11p.urgency == SourcePackageUrgency.LOW
     True
 
+Check that the changelog was uploaded to the librarian correctly:
+
+    >>> print x11p.changelog.read()
+    x11proto-damage (6.8.99.7-2) breezy; urgency=low
+    <BLANKLINE>
+      * Add dependency on x11proto-fixes-dev.
+    <BLANKLINE>
+     -- Daniel Stone <[e-mail]>  Mon, 11 Jul 2005 19:11:11 +1000
+    <BLANKLINE>
+    x11proto-damage (6.8.99.7-1) breezy; urgency=low
+    <BLANKLINE>
+      * First x11proto-damage release.
+    <BLANKLINE>
+     -- Daniel Stone <[e-mail]>  Mon, 16 May 2005 22:10:17 +1000
+
 Same for the copyright:
 
     >>> print x11p.copyright
 
=== modified file 'lib/lp/soyuz/scripts/gina/handlers.py'
--- lib/lp/soyuz/scripts/gina/handlers.py	2010-09-02 16:28:50 +0000
+++ lib/lp/soyuz/scripts/gina/handlers.py	2010-11-23 10:58:17 +0000
@@ -17,6 +17,7 @@
     'DistroHandler',
     ]
 
+from cStringIO import StringIO
 import os
 import re
 
@@ -31,6 +32,7 @@
     quote,
     sqlvalues,
     )
+from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
 from canonical.launchpad.scripts import log
 from lp.archivepublisher.diskpool import poolify
 from lp.archiveuploader.tagfiles import parse_tagfile
@@ -624,11 +626,7 @@
         to_upload = check_not_in_librarian(src.files, src.archive_root,
                                            src.directory)
 
-        #
-        # DO IT! At this point, we've decided we have everything we need
-        # to create the SPR.
-        #
-
+        # Create the SourcePackageRelease (SPR)
         componentID = self.distro_handler.getComponentByName(src.component).id
         sectionID = self.distro_handler.ensureSection(src.section).id
         maintainer_line = "%s <%s>" % (displayname, emailaddress)
@@ -645,7 +643,7 @@
             dsc=src.dsc,
             copyright=src.copyright,
             version=src.version,
-            changelog_entry=src.changelog,
+            changelog_entry=src.changelog_entry,
             builddepends=src.build_depends,
             builddependsindep=src.build_depends_indep,
             build_conflicts=src.build_conflicts,
@@ -661,6 +659,15 @@
         log.info('Source Package Release %s (%s) created' %
                  (name.name, src.version))
 
+        # Upload the changelog to the Librarian
+        if src.changelog is not None:
+            changelog_lfa = getUtility(ILibraryFileAliasSet).create(
+                "changelog",
+                len(src.changelog),
+                StringIO(src.changelog),
+                "text/x-debian-source-changelog")
+            spr.changelog = changelog_lfa
+
         # Insert file into the library and create the
         # SourcePackageReleaseFile entry on lp db.
         for fname, path in to_upload:
 
=== modified file 'lib/lp/soyuz/scripts/gina/packages.py'
--- lib/lp/soyuz/scripts/gina/packages.py	2010-10-19 11:53:02 +0000
+++ lib/lp/soyuz/scripts/gina/packages.py	2010-11-23 10:58:17 +0000
@@ -133,9 +133,7 @@
     fullpath = os.path.join(source_dir, "debian", "changelog")
     changelog = None
     if os.path.exists(fullpath):
-        clfile = open(fullpath)
-        changelog = parse_changelog(clfile)
-        clfile.close()
+        changelog = open(fullpath).read().strip()
     else:
         log.warn("No changelog file found for %s in %s" %
                  (package, source_dir))
@@ -401,11 +399,15 @@
 
         self.dsc = encoding.guess(dsc)
         self.copyright = encoding.guess(copyright)
+        parsed_changelog = None
+        if changelog:
+            parsed_changelog = parse_changelog(changelog.split('\n'))
 
         self.urgency = None
         self.changelog = None
-        if changelog and changelog[0]:
-            cldata = changelog[0]
+        self.changelog_entry = None
+        if parsed_changelog and parsed_changelog[0]:
+            cldata = parsed_changelog[0]
             if 'changes' in cldata:
                 if cldata["package"] != self.package:
                     log.warn("Changelog package %s differs from %s" %
@@ -413,7 +415,8 @@
                 if cldata["version"] != self.version:
                     log.warn("Changelog version %s differs from %s" %
                              (cldata["version"], self.version))
-                self.changelog = encoding.guess(cldata["changes"])
+                self.changelog_entry = encoding.guess(cldata["changes"])
+                self.changelog = changelog
                 self.urgency = cldata["urgency"]
             else:
                 log.warn("Changelog empty for source %s (%s)" %