Merge lp:~sergiusens/snapcraft/pull-python-projects into lp:~snappy-dev/snapcraft/core

[Sergio Schvezov]

Make the py 2 and 3 projects smarter and have dependencies pulled during the pull phase

This is a side effect of needing internet access only done during the pull phases of the lifecycle to comply with launchpad.

It would be possible if the python2 or python3 plugins could be staged before running the pull cycle for the corresponding -project ones. Since this is not a viable alternative, moving everything to the py projects is what was done.

[lp:~sergiusens/snapcraft/pull-python-projects]

201. By Sergio Schvezov on 2015-09-23

be smarter about origin of setup.py

202. By Sergio Schvezov on 2015-09-23

simplify complexity a tiny insy bit

203. By Sergio Schvezov on 2015-09-23

goof requirement

204. By Sergio Schvezov on 2015-09-23

Merging trunk

[John Lenton (chipaca)]

lgtm.

review: Approve

=== modified file 'integration-tests/data/pip-requirements/snapcraft.yaml'
--- integration-tests/data/pip-requirements/snapcraft.yaml	2015-09-09 20:19:12 +0000
+++ integration-tests/data/pip-requirements/snapcraft.yaml	2015-09-23 12:25:26 +0000
@@ -7,8 +7,8 @@
 
 parts:
   python2:
-    type: python2
+    type: python2-project
     requirements: requirements.txt
   python3:
-    type: python3
+    type: python3-project
     requirements: requirements.txt

=== modified file 'plugins/python2-project.yaml'
--- plugins/python2-project.yaml	2015-09-22 03:08:07 +0000
+++ plugins/python2-project.yaml	2015-09-23 12:25:26 +0000
@@ -1,8 +1,8 @@
-requires:
-  - python2
 options:
+  requirements:
+    required: false
   source:
-    required: true
+    required: false
   source-type:
   source-tag:
   source-branch:

=== removed file 'plugins/python2.yaml'
--- plugins/python2.yaml	2015-09-01 14:10:18 +0000
+++ plugins/python2.yaml	1970-01-01 00:00:00 +0000
@@ -1,3 +0,0 @@
-options:
-  requirements:
-    required: false

=== modified file 'plugins/python3-project.yaml'
--- plugins/python3-project.yaml	2015-09-22 03:08:07 +0000
+++ plugins/python3-project.yaml	2015-09-23 12:25:26 +0000
@@ -1,8 +1,8 @@
-requires:
-  - python3
 options:
+  requirements:
+    required: false
   source:
-    required: true
+    required: false
   source-type:
   source-tag:
   source-branch:

=== removed file 'plugins/python3.yaml'
--- plugins/python3.yaml	2015-09-09 21:35:18 +0000
+++ plugins/python3.yaml	1970-01-01 00:00:00 +0000
@@ -1,3 +0,0 @@
-options:
-  requirements:
-    required: false

=== modified file 'runtests.sh'
--- runtests.sh	2015-09-18 09:10:52 +0000
+++ runtests.sh	2015-09-23 12:25:26 +0000
@@ -54,7 +54,7 @@
 
     # mccabe in 'warning' mode as we have high complexity
     mccabe_list=
-    for unit in $(find . -type f -name '*.py')
+    for unit in $(find snapcraft -type f -name '*.py')
     do
         output=$(python3 -m mccabe --min 10 "$unit")
         [ -n "$output" ] && mccabe_list="- $unit:\n  $output\n$mccabe_list"

=== modified file 'snapcraft/cmds.py'
--- snapcraft/cmds.py	2015-09-21 19:20:43 +0000
+++ snapcraft/cmds.py	2015-09-23 12:25:26 +0000
@@ -267,6 +267,8 @@
             for f in common:
                 this = os.path.join(part.installdir, f)
                 other = os.path.join(parts_files[other_part_name]['installdir'], f)
+                if os.path.islink(this) and os.path.islink(other):
+                    continue
                 if not filecmp.cmp(this, other, shallow=False):
                     conflict_files.append(f)
 

=== removed file 'snapcraft/plugins/python2.py'
--- snapcraft/plugins/python2.py	2015-09-18 21:23:40 +0000
+++ snapcraft/plugins/python2.py	1970-01-01 00:00:00 +0000
@@ -1,63 +0,0 @@
-# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
-#
-# Copyright (C) 2015 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import snapcraft
-
-
-class Python2Plugin(snapcraft.BasePlugin):
-
-    _PLUGIN_STAGE_PACKAGES = [
-        'python-dev',
-        'python-setuptools',
-    ]
-
-    def __init__(self, name, options):
-        if options.requirements:
-            self.requirements = options.requirements
-            self._PLUGIN_STAGE_PACKAGES.extend(['python-pkg-resources', 'python-setuptools'])
-        else:
-            self.requirements = None
-        super().__init__(name, options)
-
-    def snap_fileset(self):
-        return [
-            '-usr/share',
-        ]
-
-    # note that we don't need to set PYTHONHOME here,
-    # python discovers this automatically from it installed
-    # location.  And PATH is automatically set by snapcraft.
-
-    def env(self, root):
-        return ["PYTHONPATH=%s" % os.path.join(
-            root, 'usr', 'lib', self.python_version, 'dist-packages')]
-
-    def pull(self):
-        # A nice idea here would be to be asking setup tools
-        # to use the deb layout, but that doesn't work with
-        # prefix sadly
-
-        if self.requirements and not (self.run(
-                ['ln', '-s', os.path.join(self.installdir, 'usr', 'lib', self.python_version, 'dist-packages'), os.path.join(self.installdir, 'usr', 'lib', self.python_version, 'site-packages')]) and self.run(
-                ['python2', os.path.join(self.installdir, 'usr', 'bin', 'easy_install'), '--prefix', os.path.join(self.installdir, 'usr'), 'pip']) and self.run(
-                ['python2', os.path.join(self.installdir, 'usr', 'bin', 'pip2'), 'install', '--target', os.path.join(self.installdir, 'usr', 'lib', self.python_version, 'site-packages'), '--requirement', os.path.join(os.getcwd(), self.requirements)])):
-            return False
-        return True
-
-    @property
-    def python_version(self):
-        return self.run_output(['pyversions', '-i'])

=== modified file 'snapcraft/plugins/python2_project.py'
--- snapcraft/plugins/python2_project.py	2015-09-22 23:14:22 +0000
+++ snapcraft/plugins/python2_project.py	2015-09-23 12:25:26 +0000
@@ -22,20 +22,76 @@
 
 class Python2ProjectPlugin(snapcraft.BasePlugin):
 
-    # note that we don't need to setup env(), python figures it out
-    # see python2.py for more details
+    _PLUGIN_STAGE_PACKAGES = [
+        'python-dev',
+        'python-pkg-resources',
+        'python-setuptools',
+    ]
+
+    def __init__(self, name, options):
+        super().__init__(name, options)
+        self.requirements = options.requirements
+        self.source = options.source
+
+    def env(self, root):
+        return ["PYTHONPATH=%s" % os.path.join(
+            root, 'usr', 'lib', self.python_version, 'dist-packages')]
 
     def pull(self):
-        return self.handle_source_options()
+        if self.source and not self.handle_source_options():
+            return False
+
+        return self._pip()
+
+    def _pip(self):
+        setup = 'setup.py'
+        if os.listdir(self.sourcedir):
+            setup = os.path.join(self.sourcedir, 'setup.py')
+
+        if self.requirements:
+            requirements = os.path.join(os.getcwd(), self.requirements)
+
+        if not os.path.exists(setup) and not self.requirements:
+            return True
+
+        easy_install = os.path.join(
+            self.installdir, 'usr', 'bin', 'easy_install')
+        prefix = os.path.join(self.installdir, 'usr')
+        site_packages_dir = os.path.join(
+            prefix, 'lib', self.python_version, 'site-packages')
+
+        if not os.path.exists(site_packages_dir):
+            os.symlink(
+                os.path.join(prefix, 'lib', self.python_version, 'dist-packages'),
+                site_packages_dir)
+
+        if not self.run(['python2', easy_install, '--prefix', prefix, 'pip']):
+            return False
+
+        pip2 = os.path.join(self.installdir, 'usr', 'bin', 'pip2')
+        pip_install = ['python2', pip2, 'install', '--target',
+                       site_packages_dir]
+
+        if self.requirements and not self.run(
+                pip_install + ['--requirement', requirements]):
+            return False
+
+        if os.path.exists(setup) and not self.run(pip_install + ['.', ]):
+            return False
+
+        return True
 
     def build(self):
         # If setuptools is used, it tries to create files in the
         # dist-packages dir and import from there, so it needs to exist
         # and be in the PYTHONPATH. It's harmless if setuptools isn't
         # used.
+
+        if not os.path.exists(os.path.join(self.builddir, 'setup.py')):
+            return True
+
         os.makedirs(self.dist_packages_dir, exist_ok=True)
         setuptemp = self.copy_setup()
-
         return self.run(
             ['python2', setuptemp.name, 'install', '--install-layout=deb',
              '--prefix={}/usr'.format(self.installdir)], cwd=self.builddir)

=== removed file 'snapcraft/plugins/python3.py'
--- snapcraft/plugins/python3.py	2015-09-18 21:23:40 +0000
+++ snapcraft/plugins/python3.py	1970-01-01 00:00:00 +0000
@@ -1,65 +0,0 @@
-# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
-#
-# Copyright (C) 2015 Canonical Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import snapcraft
-
-
-class Python3Plugin(snapcraft.BasePlugin):
-
-    _PLUGIN_STAGE_PACKAGES = [
-        'python3-dev',
-        'python3-setuptools',
-    ]
-
-    def __init__(self, name, options):
-        if options.requirements:
-            self.requirements = options.requirements
-            self._PLUGIN_STAGE_PACKAGES.extend(['python3-pkg-resources', 'python3-setuptools'])
-        else:
-            self.requirements = None
-        super().__init__(name, options)
-
-    def snap_fileset(self):
-        return [
-            '-usr/share',
-        ]
-
-    # note that we don't need to set PYTHONHOME here,
-    # python discovers this automatically from it installed
-    # location, see https://code.launchpad.net/~mvo/snapcraft/python3-project/+merge/264521/comments/664308
-    #
-    # PATH is automatically set by snapcraft
-
-    def env(self, root):
-        return ["PYTHONPATH=%s" % os.path.join(
-            root, 'usr', 'lib', 'python3', 'dist-packages')]
-
-    def pull(self):
-        # A nice idea here would be to be asking setup tools
-        # to use the deb layout, but that doesn't work with
-        # prefix sadly
-
-        if self.requirements and not (self.run(
-                ['ln', '-s', os.path.join(self.installdir, 'usr', 'lib', 'python3', 'dist-packages'), os.path.join(self.installdir, 'usr', 'lib', self.python_version, 'site-packages')]) and self.run(
-                ['python3', os.path.join(self.installdir, 'usr', 'bin', 'easy_install3'), '--prefix', os.path.join(self.installdir, 'usr'), 'pip']) and self.run(
-                ['python3', os.path.join(self.installdir, 'usr', 'bin', 'pip3'), 'install', '--target', os.path.join(self.installdir, 'usr', 'lib', self.python_version, 'site-packages'), '--requirement', os.path.join(os.getcwd(), self.requirements)])):
-            return False
-        return True
-
-    @property
-    def python_version(self):
-        return self.run_output(['py3versions', '-i'])

=== modified file 'snapcraft/plugins/python3_project.py'
--- snapcraft/plugins/python3_project.py	2015-09-21 22:22:36 +0000
+++ snapcraft/plugins/python3_project.py	2015-09-23 12:25:26 +0000
@@ -22,21 +22,74 @@
 
 class Python3ProjectPlugin(snapcraft.BasePlugin):
 
-    # note that we don't need to setup env(), python figures it out
-    # see python3.py for more details
+    _PLUGIN_STAGE_PACKAGES = [
+        'python3-dev',
+        'python3-pkg-resources',
+        'python3-setuptools',
+    ]
+
+    def __init__(self, name, options):
+        super().__init__(name, options)
+        self.requirements = options.requirements
+        self.source = options.source
 
     def env(self, root):
         return ["PYTHONPATH=%s" % os.path.join(
-            root, 'usr', 'lib', 'python3', 'dist-packages')]
+            root, 'usr', 'lib', self.python_version, 'dist-packages')]
 
     def pull(self):
-        return self.handle_source_options()
+        if self.source and not self.handle_source_options():
+            return False
+
+        return self._pip()
+
+    def _pip(self):
+        setup = 'setup.py'
+        if os.listdir(self.sourcedir):
+            setup = os.path.join(self.sourcedir, 'setup.py')
+
+        if self.requirements:
+            requirements = os.path.join(os.getcwd(), self.requirements)
+
+        if not os.path.exists(setup) and not self.requirements:
+            return True
+
+        easy_install = os.path.join(
+            self.installdir, 'usr', 'bin', 'easy_install3')
+        prefix = os.path.join(self.installdir, 'usr')
+        site_packages_dir = os.path.join(
+            prefix, 'lib', self.python_version, 'site-packages')
+
+        if not os.path.exists(site_packages_dir):
+            os.symlink(
+                os.path.join(prefix, 'lib', 'python3', 'dist-packages'),
+                site_packages_dir)
+
+        if not self.run(['python3', easy_install, '--prefix', prefix, 'pip']):
+            return False
+
+        pip3 = os.path.join(self.installdir, 'usr', 'bin', 'pip3')
+        pip_install = ['python3', pip3, 'install', '--target',
+                       site_packages_dir]
+
+        if self.requirements and not self.run(
+                pip_install + ['--requirement', requirements]):
+            return False
+
+        if os.path.exists(setup) and not self.run(pip_install + ['.', ]):
+            return False
+
+        return True
 
     def build(self):
         # If setuptools is used, it tries to create files in the
         # dist-packages dir and import from there, so it needs to exist
         # and be in the PYTHONPATH. It's harmless if setuptools isn't
         # used.
+
+        if not os.path.exists(os.path.join(self.builddir, 'setup.py')):
+            return True
+
         os.makedirs(self.dist_packages_dir, exist_ok=True)
         setuptemp = self.copy_setup()
         return self.run(
@@ -46,7 +99,11 @@
     @property
     def dist_packages_dir(self):
         return os.path.join(
-            self.installdir, 'usr', 'lib', 'python3', 'dist-packages')
+            self.installdir, 'usr', 'lib', self.python_version, 'dist-packages')
+
+    @property
+    def python_version(self):
+        return self.run_output(['py3versions', '-i'])
 
     # Takes the setup.py file and puts a couple little gems on the
     # front to make things work better.