Merge lp:~aptdaemon-developers/aptdaemon/lintian-profiles into lp:aptdaemon

[Sebastian Heinlein]

Use lintian profiles instead of the tag files

[lp:~aptdaemon-developers/aptdaemon/lintian-profiles]

840. By Sebastian Heinlein on 2012-06-09

Set $HOME to a temporary dir before executing lintian

Setting LINTIAN_INTERNAL_TESTSUITE doesn't load any custom profiles
from /etc and would not allow admins to modify the installed policy easily

841. By Sebastian Heinlein on 2012-06-11

Push local changes

842. By Sebastian Heinlein on 2012-06-11

Merge with trunk

843. By Sebastian Heinlein on 2012-06-11

Update NEWS

[Michael Vogt (mvo)]

This looks fine.

review: Approve

=== modified file 'NEWS'
--- NEWS	2012-06-11 06:38:54 +0000
+++ NEWS	2012-06-11 06:52:21 +0000
@@ -8,6 +8,9 @@
 
    - Support more PackageKit transactions (InstallSignature, RepoEnable)
 
+   - Use lintian profiles instead of tag files (separat profiles for Debian
+     and Ubuntu are available). Thanks to Niels Thykier.
+
  * API changes:
 
    - aptdaemon.test.TestCase has now a start_keyserver() method to fake

=== modified file 'aptdaemon/progress.py'
--- aptdaemon/progress.py	2012-06-07 04:59:57 +0000
+++ aptdaemon/progress.py	2012-06-11 06:52:21 +0000
@@ -20,8 +20,11 @@
 import logging
 import os
 import re
+import shutil
 import signal
+import subprocess
 import sys
+import tempfile
 import termios
 import time
 import traceback
@@ -609,6 +612,48 @@
         return False
 
 
+class DaemonLintianProgress(DaemonForkProgress):
+
+    """Performs a lintian call."""
+
+    def start_update(self):
+        DaemonForkProgress.start_update(self)
+        self.tempdir = os.getenv("APTDAEMON_LINTIAN_HOME",
+                                 tempfile.mkdtemp(prefix="aptdaemon-lintian-"))
+
+    def finish_update(self):
+        DaemonForkProgress.finish_update(self)
+        if not os.getenv("APTDAEMON_LINTIAN_HOME"):
+            try:
+                shutil.rmtree(self.tempdir)
+            except AttributeError:
+                pass
+
+    def _child(self, path):
+        # Avoid running lintian as root
+        os.setuid(self.transaction.uid)
+        # FIXME: Should be handled by lintian ideally
+        if subprocess.check_call(["/usr/bin/dpkg-vendor",
+                                  "--derives-from", "Ubuntu"]) == 0:
+            profile = "ubuntu/aptdaemon"
+        else:
+            profile = "debian/aptdaemon"
+        # By default lintian tries to load profiles from
+        # $HOME/.lintian/profiles. This would allow an user to inject
+        # a relaxed profile.
+        # Currently the loading of custom profiles can only disabled by
+        # setting the LINTIAN_INTERNAL_TESTSUITE env.
+        # But then also profiles from /etc/lintian would be skipped
+        # See, LP# 1006327
+        os.environ["HOME"] = self.tempdir
+
+        lintian_path = apt_pkg.config.find_file("Dir::Bin::Lintian",
+                                                "/usr/bin/lintian")
+        os.execlp(lintian_path, lintian_path, "--no-cfg", "--fail-on-warnings",
+                   "--profile", profile, path)
+        os._exit(1)
+
+
 class DaemonInstallProgress(DaemonForkProgress):
 
     """Progress to execute APT package operations in a child process."""

=== modified file 'aptdaemon/worker.py'
--- aptdaemon/worker.py	2012-06-08 05:26:16 +0000
+++ aptdaemon/worker.py	2012-06-11 06:52:21 +0000
@@ -56,6 +56,7 @@
                      DaemonDpkgInstallProgress, \
                      DaemonDpkgReconfigureProgress, \
                      DaemonDpkgRecoverProgress, \
+                     DaemonLintianProgress, \
                      DaemonForkProgress
 
 log = logging.getLogger("AptDaemon.Worker")
@@ -596,7 +597,7 @@
         log.info("Installing local package file: %s", path)
         # Check if the dpkg can be installed at all
         trans.status = STATUS_RESOLVING_DEP
-        deb = self._check_deb_file(path, force, trans.uid)
+        deb = self._check_deb_file(trans, path, force)
         # Check for required changes and apply them before
         (install, remove, unauth) = deb.required_changes
         self._call_plugins("modify_cache_after")
@@ -1172,57 +1173,23 @@
 
         return depends, required_download, required_space, unauthenticated
 
-    def _check_deb_file(self, path, force, uid):
+    def _check_deb_file(self, trans, path, force):
         """Perform some basic checks for the Debian package.
 
         :param trans: The transaction instance.
 
         :returns: An apt.debfile.Debfile instance.
         """
-        #FIXME: Unblock lintian call
         if not os.path.isfile(path):
             raise TransactionFailed(ERROR_UNREADABLE_PACKAGE_FILE, path)
         if not force and os.path.isfile("/usr/bin/lintian"):
-            tags_dir = os.path.join(apt_pkg.config.find_dir("Dir"),
-                                    "usr", "share", "aptdaemon")
-            try:
-                distro = platform.dist()[0]
-            except KeyError:
-                distro = None
-            else:
-                tags_file = os.path.join(tags_dir,
-                                         "lintian-nonfatal.tags.%s" % distro)
-                tags_fatal_file = os.path.join(tags_dir,
-                                               "lintian-fatal.tags.%s" % distro)
-            if not distro or not os.path.exists(tags_file):
-                log.debug("Using default lintian tags file")
-                tags_file = os.path.join(tags_dir, "lintian-nonfatal.tags")
-            if not distro or not os.path.exists(tags_fatal_file):
-                log.debug("Using default lintian fatal tags file")
-                tags_fatal_file = os.path.join(tags_dir, "lintian-fatal.tags")
-            # Run linitan as the user who initiated the transaction
-            # Once with non fatal checks and a second time with the fatal
-            # checks which are not allowed to be overriden
-            nonfatal_args = ["/usr/bin/lintian", "--tags-from-file",
-                             tags_file, path]
-            fatal_args = ["/usr/bin/lintian", "--tags-from-file",
-                          tags_fatal_file, "--no-override", path]
-            for lintian_args in (nonfatal_args, fatal_args):
-                proc = subprocess.Popen(lintian_args,
-                                        stderr=subprocess.STDOUT,
-                                        stdout=subprocess.PIPE, close_fds=True,
-                                        preexec_fn=lambda: os.setuid(uid))
-                while proc.poll() is None:
-                    self._iterate_mainloop()
-                    time.sleep(0.05)
-                #FIXME: Add an error to catch return state 2 (failure)
-                if proc.returncode == 1:
-                    stdout = proc.stdout.read()
-                    stdout.decode(sys.stdin.encoding or "UTF-8",
-                                  errors="replace")
-                    raise TransactionFailed(ERROR_INVALID_PACKAGE_FILE,
-                                            "Lintian check results for %s:"
-                                            "\n%s" % (path, stdout))
+            with DaemonLintianProgress(trans) as progress:
+                progress.run(path)
+            #FIXME: Add an error to catch return state 2 (failure)
+            if progress._child_exit != 0:
+                raise TransactionFailed(ERROR_INVALID_PACKAGE_FILE,
+                                        "Lintian check results for %s:"
+                                        "\n%s" % (path, progress.output))
         try:
             deb = apt.debfile.DebPackage(path, self._cache)
         except IOError:

=== added directory 'data/lintian'
=== removed file 'data/lintian-fatal.tags'
--- data/lintian-fatal.tags	2011-04-06 07:56:33 +0000
+++ data/lintian-fatal.tags	1970-01-01 00:00:00 +0000
@@ -1,66 +0,0 @@
-# This file lists all tags that cause an automatic reject on upload and cannot
-# be overridden.  It is based on the data file retrieved from
-# http://ftp-master.debian.org/static/lintian.tags
-#
-# Last updated: 2011-01-26
-
-FSSTND-dir-in-usr
-FSSTND-dir-in-var
-bad-package-name
-bad-relation
-bad-version-number
-binary-file-compressed-with-upx
-binary-in-etc
-build-info-in-binary-control-file-section
-control-file-has-bad-owner
-control-file-has-bad-permissions
-control-interpreter-in-usr-local
-copyright-file-compressed
-copyright-file-is-symlink
-copyright-refers-to-incorrect-directory
-copyright-refers-to-old-directory
-debian-control-file-uses-obsolete-national-encoding
-debian-control-with-duplicate-fields
-debian-rules-missing-required-target
-debian-rules-not-a-makefile
-description-is-dh_make-template
-description-synopsis-is-empty
-dir-or-file-in-mnt
-# Allow third party packages to make use of /opt
-# dir-or-file-in-opt
-dir-or-file-in-srv
-dir-or-file-in-tmp
-extended-description-is-empty
-file-in-etc-not-marked-as-conffile
-file-in-usr-marked-as-conffile
-forbidden-postrm-interpreter
-library-in-debug-or-profile-should-not-be-stripped
-magic-arch-in-arch-list
-maintainer-address-is-on-localhost
-maintainer-address-malformed
-maintainer-address-missing
-maintainer-name-missing
-maintainer-script-removes-device-files
-malformed-deb-archive
-missing-dependency-on-perlapi
-no-architecture-field
-no-copyright-file
-no-maintainer-field
-no-package-name
-no-source-field
-no-version-field
-not-allowed-control-file
-old-style-copyright-file
-package-contains-ancient-file
-package-has-no-description
-package-installs-python-bytecode
-package-not-lowercase
-package-uses-local-diversion
-section-is-dh_make-template
-source-field-does-not-match-pkg-name
-symlink-has-too-many-up-segments
-too-many-architectures
-uploader-address-is-on-localhost
-uploader-address-malformed
-uploader-name-missing
-usr-share-doc-symlink-to-foreign-package

=== removed file 'data/lintian-nonfatal.tags'
--- data/lintian-nonfatal.tags	2011-04-15 07:50:24 +0000
+++ data/lintian-nonfatal.tags	1970-01-01 00:00:00 +0000
@@ -1,25 +0,0 @@
-# This file lists all tags that cause an automatic reject on upload but can
-# be overridden (nonfatal tags).  It is based on the data file retrieved from
-# http://ftp-master.debian.org/static/lintian.tags
-#
-# Last updated: 2011-01-26
-
-arch-dependent-file-in-usr-share
-arch-independent-package-contains-binary-or-object
-binary-or-shlib-defines-rpath
-binary-with-bad-dynamic-table
-control-interpreter-without-depends
-copyright-contains-dh_make-todo-boilerplate
-dir-or-file-in-var-www
-embedded-library
-missing-build-dependency
-# For LSB packages it is sufficient to depend on lsb-base
-# missing-dependency-on-libc
-mknod-in-maintainer-script
-no-shlibs-control-file
-non-etc-file-marked-as-conffile
-package-contains-info-dir-file
-preinst-interpreter-without-predepends
-statically-linked-binary
-usr-share-doc-symlink-without-dependency
-wrong-file-owner-uid-or-gid

=== added directory 'data/lintian/debian'
=== added file 'data/lintian/debian/aptdaemon.profile'
--- data/lintian/debian/aptdaemon.profile	1970-01-01 00:00:00 +0000
+++ data/lintian/debian/aptdaemon.profile	2012-06-11 06:52:21 +0000
@@ -0,0 +1,7 @@
+# The default profile installing local software packages used by aptdaemon
+Profile: debian/aptdaemon
+Extends: debian/ftp-master-auto-reject
+Disable-Tags:
+	dir-or-file-in-opt,
+	missing-dependency-on-libc,
+	statically-linked-binary

=== added directory 'data/lintian/ubuntu'
=== added file 'data/lintian/ubuntu/aptdaemon.profile'
--- data/lintian/ubuntu/aptdaemon.profile	1970-01-01 00:00:00 +0000
+++ data/lintian/ubuntu/aptdaemon.profile	2012-06-11 06:52:21 +0000
@@ -0,0 +1,43 @@
+# The default profile installing local software packages used by aptdaemon
+Profile: ubuntu/aptdaemon
+Extends: debian/aptdaemon
+Disable-Tags:
+	FSSTND-dir-in-usr,
+	FSSTND-dir-in-var,
+	arch-dependent-file-in-usr-share,
+        binary-file-compressed-with-upx,
+	binary-or-shlib-defines-rpath,
+        control-interpreter-in-usr-local,
+	copyright-contains-dh_make-todo-boilerplate,
+	copyright-file-compressed,
+ 	copyright-file-is-symlink,
+	copyright-refers-to-incorrect-directory,
+	copyright-refers-to-old-directory,
+ 	debian-rules-missing-required-target,
+	debian-rules-not-a-makefile,
+	description-is-dh_make-template,
+	description-synopsis-is-empty,
+	dir-or-file-in-srv,
+	dir-or-file-in-var-www,
+	embedded-library,
+	extended-description-is-empty,
+	file-in-usr-marked-as-conffile,
+	install-info-used-in-maintainer-script,
+ 	library-in-debug-or-profile-should-not-be-stripped,
+	magic-arch-in-arch-list,
+	missing-build-dependency,
+	missing-pre-dependency-on-multiarch-support,
+	mknod-in-maintainer-script,
+	no-architecture-field,
+	no-copyright-file,
+ 	no-maintainer-field,
+	no-shlibs-control-file,
+	package-installs-python-bytecode,
+	section-is-dh_make-template,
+	statically-linked-binary,
+	udeb-uses-unsupported-compression-for-data-tarball,
+ 	uploader-address-is-on-localhost,
+	uploader-address-malformed,
+	uploader-name-missing,
+	usr-share-doc-symlink-to-foreign-package,
+	usr-share-doc-symlink-without-dependency,

=== modified file 'setup.py'
--- setup.py	2012-06-04 07:28:07 +0000
+++ setup.py	2012-06-11 06:52:21 +0000
@@ -46,11 +46,13 @@
                    ["data/20dbus"]),
                   ("share/apport/package-hooks",
                    ["apport/aptdaemon.py"]),
-                  ("share/aptdaemon",
-                   glob.glob("data/lintian-*fatal.tags*")),
                   ("share/dbus-1/system-services/",
                    ["data/org.debian.apt.service",
                     "data/org.freedesktop.PackageKit.service"]),
+                  ("share/lintian/profiles/ubuntu/",
+                   ["data/lintian/ubuntu/aptdaemon.profile"]),
+                  ("share/lintian/profiles/debian/",
+                   ["data/lintian/debian/aptdaemon.profile"]),
                   ("share/man/man1",
                    ["doc/aptd.1", "doc/aptdcon.1"]),
                   ("share/man/man7",

=== modified file 'tests/test_worker.py'
--- tests/test_worker.py	2012-06-04 09:17:34 +0000
+++ tests/test_worker.py	2012-06-11 06:52:21 +0000
@@ -288,15 +288,19 @@
     def test_install_file(self):
         """Test the installation of a local package file."""
         # add custom lintian file
-        target = os.path.join(self.chroot.path, "usr", "share", "aptdaemon")
-        os.makedirs(target)
-        for tags_file in glob.glob(os.path.join(aptdaemon.test.get_tests_dir(),
-                                                "../data/lintian*tags*")):
-            shutil.copy(tags_file, target)
+        os.environ["APTDAEMON_LINTIAN_HOME"] = self.chroot.path
+        self.addCleanup(os.putenv, "APTDAEMON_LINTIAN_HOME", "")
+        os.makedirs(os.path.join(self.chroot.path, ".lintian/profiles/"))
+        for profile in glob.glob(os.path.join(aptdaemon.test.get_tests_dir(),
+                                                "../data/lintian/*")):
+            shutil.copytree(profile,
+                            os.path.join(self.chroot.path,
+                                         ".lintian/profiles",
+                                         os.path.basename(profile)))
         # test
         self.chroot.add_test_repository()
-        pkg = os.path.join(
-            REPO_PATH, "silly-depend-base-lintian-broken_0.1-0_all.deb")
+        pkg = os.path.join(REPO_PATH,
+                           "silly-depend-base-lintian-broken_0.1-0_all.deb")
         trans = Transaction(None, enums.ROLE_INSTALL_FILE, self.queue,
                             os.getpid(), os.getuid(), sys.argv[0],
                             "org.debian.apt.test", connect=False,
@@ -304,6 +308,7 @@
                                     "force": False})
         self.worker.simulate(trans)
         self.loop.run()
+        self.assertTrue("wrong-file-owner-uid-or-gid" in trans.error.details)
         self.assertEqual(trans.error.code, enums.ERROR_INVALID_PACKAGE_FILE,
                          "Lintian failed to detect a broken package")
         # Now allow to install invalid packages