Merge lp:~cjwatson/launchpad-buildd/normalise-python-packaging into lp:launchpad-buildd

[Colin Watson]

Normalise Python packaging. We now install our modules on the normal system path, using pybuild. setup.py now installs buildd-slave.tac in the lpbuildd package rather than data_files in order not to pollute the top level of a virtualenv.

I haven't tested it yet, but this should hopefully help to get buildd-related tests passing in my LP virtualenv-pip branch.

[William Grant (wgrant)]

review: Approve (code)

=== modified file '.bzrignore'
--- .bzrignore	2015-11-04 14:10:28 +0000
+++ .bzrignore	2017-09-08 17:28:12 +0000
@@ -1,9 +1,13 @@
 *.egg-info
 *.pyc
+.pybuild
+buildd-slave-example.conf
 dist
+debian/debhelper-build-stamp
 debian/files
 debian/launchpad-buildd
 debian/python-lpbuildd
+debian/tmp
 debian/*.debhelper.log
 debian/*.debhelper
 debian/*.substvars

=== modified file 'MANIFEST.in'
--- MANIFEST.in	2017-09-08 17:28:12 +0000
+++ MANIFEST.in	2017-09-08 17:28:12 +0000
@@ -8,8 +8,7 @@
 include bin/test_buildd_generatetranslationtemplates
 include bin/test_buildd_recipe
 include buildd-genconfig
-include buildd-slave.tac
 include debian/changelog
 include sbuildrc
 include template-buildd-slave.conf
-recursive-include lpbuildd/tests *.diff *.tar.gz buildd-slave-test.conf buildlog buildlog.long
+recursive-include lpbuildd/tests *.diff *.tar.gz buildd-slave.tac buildd-slave-test.conf buildlog buildlog.long

=== modified file 'debian/changelog'
--- debian/changelog	2017-09-08 17:28:12 +0000
+++ debian/changelog	2017-09-08 17:28:12 +0000
@@ -4,6 +4,10 @@
   * Merge TranslationTemplatesBuildState.{INSTALL,GENERATE} into a single
     state.
   * Convert generate-translation-templates to the new Operation framework.
+  * Normalise Python packaging.  We now install our modules on the normal
+    system path, using pybuild.  setup.py now installs buildd-slave.tac in
+    the lpbuildd package rather than data_files in order not to pollute the
+    top level of a virtualenv.
 
  -- Colin Watson <[e-mail]>  Fri, 08 Sep 2017 13:42:17 +0100
 

=== modified file 'debian/compat'
--- debian/compat	2015-08-25 09:23:24 +0000
+++ debian/compat	2017-09-08 17:28:12 +0000
@@ -1,1 +1,1 @@
-7
+9

=== modified file 'debian/control'
--- debian/control	2017-08-25 13:34:14 +0000
+++ debian/control	2017-09-08 17:28:12 +0000
@@ -3,14 +3,50 @@
 Priority: extra
 Maintainer: Adam Conrad <[e-mail]>
 Standards-Version: 3.9.5
-Build-Depends: debhelper (>= 7.0.50~), apt-utils, bzr, intltool, python, python-apt, python-debian, python-fixtures, python-mock, python-netaddr, python-pylxd, python-setuptools, python-systemfixtures, python-testtools, python-twisted, python-txfixtures, python-zope.interface
+Build-Depends: apt-utils,
+               bzr,
+               debhelper (>= 9~),
+               dh-exec,
+               dh-python,
+               intltool,
+               python (>= 2.6.6-3~),
+               python-apt,
+               python-debian,
+               python-fixtures,
+               python-mock,
+               python-netaddr,
+               python-pylxd,
+               python-setuptools,
+               python-systemfixtures,
+               python-testtools,
+               python-twisted,
+               python-txfixtures,
+# We don't use the bits of pylxd that require this at run-time, but at
+# build-time pybuild fails if it's not available.
+               python-ws4py,
+               python-zope.interface,
 
 Package: launchpad-buildd
 Section: misc
 Architecture: all
-Depends: python-lpbuildd (=${source:Version}), python, debootstrap, dpkg-dev,
- file, bzip2, sudo, ntpdate, adduser, apt-transport-https, lsb-release,
- pristine-tar, python-apt, sbuild, schroot, lsof, lxd, ${misc:Depends}
+Depends: adduser,
+         apt-transport-https,
+         bzip2,
+         debootstrap,
+         dpkg-dev,
+         file,
+         lsb-release,
+         lsof,
+         lxd,
+         ntpdate,
+         pristine-tar,
+         python,
+         python-apt,
+         python-lpbuildd (=${source:Version}),
+         sbuild,
+         schroot,
+         sudo,
+         ${misc:Depends},
 Description: Launchpad buildd slave
  This is the launchpad buildd slave package. It contains everything needed to
  get a launchpad buildd going apart from the database manipulation required to
@@ -21,7 +57,13 @@
 Package: python-lpbuildd
 Section: python
 Architecture: all
-Depends: python, python-twisted-core, python-twisted-web, python-zope.interface, python-apt, python-debian (>= 0.1.23), python-netaddr, python-pylxd, python-setuptools, apt-utils, procps, ${misc:Depends}
+Depends: apt-utils,
+         procps,
+         python-apt,
+# Work around missing dependency in python-pbr.
+         python-setuptools,
+         ${misc:Depends},
+         ${python:Depends},
 Breaks: launchpad-buildd (<< 88)
 Replaces: launchpad-buildd (<< 88)
 Description: Python libraries for a Launchpad buildd slave

=== modified file 'debian/launchpad-buildd.install'
--- debian/launchpad-buildd.install	2017-09-08 17:28:12 +0000
+++ debian/launchpad-buildd.install	2017-09-08 17:28:12 +0000
@@ -1,10 +1,10 @@
-default/launchpad-buildd 		etc/default
-buildd-genconfig			usr/share/launchpad-buildd
-debian/upgrade-config			usr/share/launchpad-buildd
-sbuildrc				usr/share/launchpad-buildd
-template-buildd-slave.conf		usr/share/launchpad-buildd
 bin/buildrecipe				usr/share/launchpad-buildd/slavebin
 bin/in-target				usr/share/launchpad-buildd/slavebin
 bin/sbuild-package			usr/share/launchpad-buildd/slavebin
 bin/slave-prep				usr/share/launchpad-buildd/slavebin
 bin/snap-git-proxy			usr/share/launchpad-buildd/slavebin
+buildd-genconfig			usr/share/launchpad-buildd
+debian/upgrade-config			usr/share/launchpad-buildd
+default/launchpad-buildd 		etc/default
+sbuildrc				usr/share/launchpad-buildd
+template-buildd-slave.conf		usr/share/launchpad-buildd

=== added file 'debian/pydist-overrides'
--- debian/pydist-overrides	1970-01-01 00:00:00 +0000
+++ debian/pydist-overrides	2017-09-08 17:28:12 +0000
@@ -0,0 +1,1 @@
+python-debian python-debian (>= 0.1.23)

=== removed file 'debian/python-lpbuildd.install'
--- debian/python-lpbuildd.install	2017-07-28 20:33:31 +0000
+++ debian/python-lpbuildd.install	1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-buildd-slave.tac 		usr/lib/launchpad-buildd
-lpbuildd/*py			usr/lib/launchpad-buildd/lpbuildd
-lpbuildd/pottery/*py		usr/lib/launchpad-buildd/lpbuildd/pottery
-lpbuildd/target/*py		usr/lib/launchpad-buildd/lpbuildd/target
-lpbuildd/tests/*		usr/lib/launchpad-buildd/lpbuildd/tests

=== added file 'debian/python-lpbuildd.links'
--- debian/python-lpbuildd.links	1970-01-01 00:00:00 +0000
+++ debian/python-lpbuildd.links	2017-09-08 17:28:12 +0000
@@ -0,0 +1,2 @@
+#! /usr/bin/dh-exec
+${LIBDIR}/lpbuildd/buildd-slave.tac	usr/lib/launchpad-buildd/buildd-slave.tac

=== modified file 'debian/rules'
--- debian/rules	2017-07-27 15:35:27 +0000
+++ debian/rules	2017-09-08 17:28:12 +0000
@@ -1,24 +1,27 @@
 #!/usr/bin/make -f
 #
-# Copyright 2009-2015 Canonical Ltd.  
+# Copyright 2009-2017 Canonical Ltd.  
 # 
 # This software is licensed under the GNU Affero General Public License version
 # 3 (see the file LICENSE).
 
+export PYBUILD_NAME := lpbuildd
+
+export LIBDIR := $(shell python -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_lib())')
+
 %:
-	dh $@
+	dh $@ --with=python2 --buildsystem=pybuild
 
 override_dh_auto_build:
+	dh_auto_build
 	python buildd-genconfig --template=template-buildd-slave.conf \
 	--arch=i386 --port=8221 --name=default --host=buildd.buildd \
 		> buildd-slave-example.conf
 
-override_dh_auto_test:
-	dh_auto_test
-	find -name \*.pyc -print0 | xargs -0r rm -f
-
-override_dh_auto_clean:
-	find -name \*.pyc -print0 | xargs -0r rm -f
+override_dh_auto_install:
+	dh_auto_install
+	# This should be split into a testable library and a wrapper.
+	chmod +x debian/python-lpbuildd/$(LIBDIR)/lpbuildd/check_implicit_pointer_functions.py
 
 override_dh_builddeb:
 	dh_builddeb -- -Zgzip

=== renamed file 'buildd-slave.tac' => 'lpbuildd/buildd-slave.tac'
=== modified file 'lpbuildd/tests/harness.py'
--- lpbuildd/tests/harness.py	2017-08-03 10:16:57 +0000
+++ lpbuildd/tests/harness.py	2017-09-08 17:28:12 +0000
@@ -125,7 +125,6 @@
         return os.path.abspath(os.path.join(
             os.path.dirname(__file__),
             os.path.pardir,
-            os.path.pardir,
             'buildd-slave.tac'
             ))
 

=== modified file 'setup.py'
--- setup.py	2017-07-27 16:12:45 +0000
+++ setup.py	2017-09-08 17:28:12 +0000
@@ -49,29 +49,26 @@
         """).strip(),
     url='https://launchpad.net/launchpad-buildd',
     packages=find_packages(),
-    package_data={
-        'lpbuildd': [
-            'tests/buildd-slave-test.conf',
-            ],
-        },
+    include_package_data=True,
     maintainer='Launchpad Developers',
     maintainer_email='[e-mail]',
     license='Affero GPL v3',
     install_requires=[
+        'netaddr',
+        'pylxd',
         # XXX cjwatson 2015-11-04: This does in fact require python-apt, but
         # that's normally shipped as a system package and specifying it here
         # causes problems for Launchpad's build system.
         #'python-apt',
-        'python-debian',
+        'python-debian>=0.1.23',
         'Twisted',
         'zope.interface',
         ],
-    data_files=[
-        ('', ['buildd-slave.tac', 'template-buildd-slave.conf']),
-        ],
     test_suite='lpbuildd.tests',
     tests_require=[
         'fixtures',
+        'mock',
+        'systemfixtures',
         'testtools',
         'txfixtures',
         ],