Merge lp:~ted/snapcraft/aws-iot into lp:~snappy-dev/snapcraft/core

[Ted Gould]

Add plugins for working with AWS' IoT features

[lp:~ted/snapcraft/aws-iot]

252. By Ted Gould on 2015-10-28

Fix function call

[Sergio Schvezov (sergiusens)]

Looks good, I don't follow, can I see an example for this? I guess that adding the docstring like I ask for can clarify lots here.

review: Needs Fixing

[lp:~ted/snapcraft/aws-iot]

253. By Ted Gould on 2015-10-29

Testing fixes from Sergio

254. By Ted Gould on 2015-10-29

Adding docstrings

255. By Ted Gould on 2015-10-29

Making sure we get more files

[Sergio Schvezov (sergiusens)]

This looks really good but the need for AWSCLIPlugin I want to avoid.

I still strongly believe that the AWSCLIPlugin should not be needed and a build-package for awscli should be used (we can update the deb in our ppa so we have the IOT option if need be).

Or is there a reason for the aws cli to live in the snap?

There are also some inline comments.

ftr,
$ rmadison awscli
 awscli | 1.2.9-2 | trusty/universe | source, all
 awscli | 1.7.0-1 | vivid/universe | source, all
 awscli | 1.7.0-1build1 | wily/universe | source, all
 awscli | 1.7.0-1build1 | xenial/universe | source, all

review: Needs Fixing

[lp:~ted/snapcraft/aws-iot]

256. By Ted Gould on 2015-10-29

Removing unneeded None

[Ted Gould (ted)]

On Thu, 2015-10-29 at 11:48 +0000, Sergio Schvezov wrote:
> > + def _keys_from_local(self, certsdir):
> > + certsjson = None
>
> this shouldn't be needed.
Python is weird. r256
> > === modified file 'snapcraft/plugins/python3.py'
> > --- snapcraft/plugins/python3.py 2015-10-27 19:23:02 +0000
> > +++ snapcraft/plugins/python3.py 2015-10-29 04:34:48 +0000
> > @@ -48,6 +48,15 @@
> > schema['properties']['requirements'] = {
> > 'type': 'string',
> > }
> > + schema['properties']['pip-packages'] = {
> >

>
>
> doesn't this fix the other bug about list of requirements?
>

No, because there are apparently some projects (plainbox) that use
multiple files of requirements. So there's another feature there.

Unmerged revisions

256. By Ted Gould on 2015-10-29

Removing unneeded None

255. By Ted Gould on 2015-10-29

Making sure we get more files

254. By Ted Gould on 2015-10-29

Adding docstrings

253. By Ted Gould on 2015-10-29

Testing fixes from Sergio

252. By Ted Gould on 2015-10-28

Fix function call

251. By Ted Gould on 2015-10-28

Adding the plugins to the command tests

250. By Ted Gould on 2015-10-28

Switching to the 'new run'

249. By Ted Gould on 2015-10-28

Merge trunk

248. By Ted Gould on 2015-10-28

Reduce complexity of cert gen

247. By Ted Gould on 2015-10-28

Include cleanups

=== modified file 'snapcraft/cmds.py'
--- snapcraft/cmds.py	2015-10-26 16:35:16 +0000
+++ snapcraft/cmds.py	2015-10-29 14:24:50 +0000
@@ -273,10 +273,7 @@
     parts_files = {}
     for part in parts:
         # Gather our own files up
-        fileset = getattr(part.code.options, 'stage', ['*']) or ['*']
-        part_files, _ = lifecycle.migratable_filesets(
-            fileset,
-            part.installdir)
+        part_files, _ = part.migratable_fileset_for('stage')
 
         # Scan previous parts for collisions
         for other_part_name in parts_files:

=== modified file 'snapcraft/lifecycle.py'
--- snapcraft/lifecycle.py	2015-10-26 14:04:47 +0000
+++ snapcraft/lifecycle.py	2015-10-29 14:24:50 +0000
@@ -151,11 +151,11 @@
         self.code.build()
         self.mark_done('build')
 
-    def _migratable_fileset_for(self, stage):
+    def migratable_fileset_for(self, stage):
         plugin_fileset = self.code.snap_fileset()
         fileset = getattr(self.code.options, stage, ['*']) or ['*']
         fileset.extend(plugin_fileset)
-        return migratable_filesets(fileset, self.installdir)
+        return _migratable_filesets(fileset, self.installdir)
 
     def _organize(self):
         organize_fileset = getattr(self.code.options, 'organize', {}) or {}
@@ -185,7 +185,7 @@
 
         self.notify_stage("Staging")
         self._organize()
-        snap_files, snap_dirs = self._migratable_fileset_for('stage')
+        snap_files, snap_dirs = self.migratable_fileset_for('stage')
 
         try:
             _migrate_files(snap_files, snap_dirs, self.installdir,
@@ -205,7 +205,7 @@
         self.makedirs()
 
         self.notify_stage("Snapping")
-        snap_files, snap_dirs = self._migratable_fileset_for('snap')
+        snap_files, snap_dirs = self.migratable_fileset_for('snap')
 
         try:
             _migrate_files(snap_files, snap_dirs, self.stagedir, self.snapdir)
@@ -274,7 +274,7 @@
     return PluginHandler(plugin_name, part_name, properties)
 
 
-def migratable_filesets(fileset, srcdir):
+def _migratable_filesets(fileset, srcdir):
     includes, excludes = _get_file_list(fileset)
 
     include_files = _generate_include_set(srcdir, includes)

=== added file 'snapcraft/plugins/awscli.py'
--- snapcraft/plugins/awscli.py	1970-01-01 00:00:00 +0000
+++ snapcraft/plugins/awscli.py	2015-10-29 14:24:50 +0000
@@ -0,0 +1,105 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2015 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""This plugin brings in the AWS command line interface and configures it.
+
+The AWS command line can be used for a variety of things including accessing
+s3 or setting up ec2 services. It also is used for setting IoT things. This
+plugin takes configuration values for your AWS keys, downloads and installs
+the client into your snap, and configures it with those keys.
+
+Settings for this plugin include:
+
+    - accesskeyid:
+      (string)
+      AWS Access Key to use
+    - secreetaccesskey:
+      (string)
+      AWS Secret Key
+    - region
+      (string)
+      Region of EC2 to use, defaults to us-east-1
+
+It is important to not that the key will be stored in the snap itself and
+could be harvested with appropriate tools.
+"""
+
+import os
+import os.path
+import snapcraft.plugins.python3
+
+
+class AWSCLIPlugin(snapcraft.plugins.python3.Python3Plugin):
+
+    @classmethod
+    def schema(cls):
+        schema = super().schema()
+        schema['properties']['accesskeyid'] = {
+            'type': 'string',
+            'default': ''
+        }
+        schema['properties']['secretaccesskey'] = {
+            'type': 'string',
+            'default': ''
+        }
+        schema['properties']['region'] = {
+            'type': 'string',
+            'default': 'us-east-1'
+        }
+
+        schema['required'] = ['accesskeyid', 'secretaccesskey']
+
+        return schema
+
+    def __init__(self, name, options):
+        options.source = "https://github.com/aws/aws-cli.git"
+        options.source_type = 'git'
+        super().__init__(name, options)
+
+    def build(self):
+        super().build()
+
+        aws = ['python3', os.path.join(self.installdir, 'usr', 'bin', 'aws')]
+
+        self.run(aws + ['configure',
+                        'set', 'region', self.options.region])
+        self.run(aws + ['configure',
+                        'set', 'aws_access_key_id', self.options.accesskeyid])
+        self.run(aws + ['configure',
+                        'set', 'aws_secret_access_key',
+                        self.options.secretaccesskey])
+
+    def env(self, root):
+        env = super().env(root)
+        env.extend(['AWS_ACCESS_KEY_ID=%s' % self.options.accesskeyid,
+                    'AWS_SECRET_ACCESS_KEY=%s' % self.options.secretaccesskey])
+        return env
+
+    def snap_fileset(self):
+        fileset = super().snap_fileset()
+        fileset.append('-usr/bin/pip*')
+        fileset.append('-usr/lib/python3/dist-packages/easy-install.pth')
+        fileset.append('-usr/lib/python*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*/__pycache__/*.pyc')
+        fileset.append('-usr/lib/python*/*/*/*/*/*/*/*/*/*/__pycache__/*.pyc')
+        return fileset

=== added file 'snapcraft/plugins/awsiot.py'
--- snapcraft/plugins/awsiot.py	1970-01-01 00:00:00 +0000
+++ snapcraft/plugins/awsiot.py	2015-10-29 14:24:50 +0000
@@ -0,0 +1,227 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2015 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A plugin to configure an AWS IoT thing and setup for MQTT certs
+
+The AWS IoT feature requires registration and configuration to communicate
+with the EC2 cloud. This plugin has configuration options for many of
+these settings, and some it can generate if there isn't an appropriate
+value to get. It will provide a basis for any software designed to
+communicate with the EC2 IoT interfaces.
+
+    - generatekeys
+      (string)
+      True if new keys should be generated by Amazon,
+      otherwise generate keys locally
+    - policydocument
+      (string)
+      Which policy document should be used. Optional.
+      A doc which allows all IoT will be used if not specified
+    - policyname
+      (string)
+      Which policy name should be used. Optional.
+      'PubSubToAnyTopic' will be used if not specified
+    - thing
+      (string)
+      The thing to create
+    - endpoint
+      (string)
+      AWS Endpoint to use if non-default
+
+"""
+
+import os
+import snapcraft
+import urllib.request
+import os.path
+import json
+import subprocess
+
+
+class AWSIoTPlugin(snapcraft.BasePlugin):
+
+    @classmethod
+    def schema(cls):
+        return {
+            '$schema': 'http://json-schema.org/draft-04/schema#',
+            'type': 'object',
+            'properties': {
+                'generatekeys': {
+                    'type': 'boolean',
+                    'default': True
+                },
+                'policydocument': {
+                    'type': 'string',
+                    'default': ''
+                },
+                'policyname': {
+                    'type': 'string',
+                    'default': 'PubSubToAnyTopic'
+                },
+                'thing': {
+                    'type': 'string',
+                },
+                'endpoint': {
+                    'type': 'string',
+                },
+            },
+            'required': ['thing']
+        }
+
+    def __init__(self, name, options):
+        super().__init__(name, options)
+        self.aws = ['python3',
+                    os.path.join(self.stagedir, 'usr', 'bin', 'aws'),
+                    'iot']
+
+        if (options.endpoint):
+            self.aws.extend(['--endpoint', options.endpoint])
+
+    def pull(self):
+        return True
+
+    def run_to_file(self, cmds, filename):
+        output = self.run_output(cmds, cwd=self.builddir)
+        with open(filename, 'w') as f:
+            f.write(output)
+
+    def _keys_from_aws(self, certsdir):
+        # generate new keys
+        self.run_to_file(self.aws + ['create-keys-and-certificate',
+                                     '--set-as-active'],
+                         os.path.join(certsdir, 'certs.json'))
+        # separate into different files
+        with open(os.path.join(certsdir, 'certs.json')) as data_file:
+            certsjson = json.load(data_file)
+
+        with open(os.path.join(certsdir, 'cert.pem'), 'w') as text_file:
+            text_file.write(self.data['certificatePem'])
+        with open(os.path.join(certsdir, 'privateKey.pem'), 'w') \
+                as text_file:
+            text_file.write(self.data['keyPair']['PrivateKey'])
+        with open(os.path.join(certsdir, 'publicKey.pem'), 'w') \
+                as text_file:
+            text_file.write(self.data['keyPair']['PublicKey'])
+
+        return certsjson
+
+    def _keys_from_local(self, certsdir):
+        certsjson = None
+
+        # generate private key
+        csr = os.path.join(certsdir, 'cert.csr')
+        self.run(['openssl', 'genrsa',
+                  '-out', os.path.join(certsdir, 'privateKey.pem'),
+                  '2048'])
+        self.run(['openssl', 'req', '-new',
+                  '-key', os.path.join(certsdir, 'privateKey.pem'),
+                  '-out', csr])
+
+        # generate new keys based on a csr
+        certresp = os.path.join(self.builddir, 'certresponse.txt')
+        self.run_to_file(self.aws + ['create-certificate-from-csr',
+                                     '--certificate-signing-request', csr,
+                                     '--set-as-active'],
+                         certresp)
+        with open(certresp) as data_file:
+            certsjson = json.load(data_file)
+
+        self.run_to_file(self.aws + ['describe-certificate',
+                                     '--certificate-id',
+                                     self.data['arn'].split(':cert/')[1],
+                                     '--output',
+                                     'text',
+                                     '--query',
+                                     '{}Description.{}Pem'.format(
+                                        'certificate')
+                                     ],
+                         os.path.join(certsdir, 'cert.pem'))
+
+        return certsjson
+
+    def build(self):
+        certsdir = os.path.join(self.builddir, 'certs')
+        # Make the certs directory if it does not exist
+        os.makedirs(certsdir, exist_ok=True)
+
+        # What should we do with certificates?
+        if self.options.generatekeys:
+            self.data = self._keys_from_aws(certsdir)
+        else:
+            self.data = self._keys_from_local(certsdir)
+
+        # Extra check, but good to ensure
+        if self.data is None:
+            return
+
+        # Get the root certificate
+        self.filename = urllib.request.urlretrieve(
+            'https://www.symantec.com/content/en/us/enterprise/verisign/roots/'
+            'VeriSign-Class%203-Public-Primary-Certification-Authority-G5.pem',
+            filename=os.path.join(certsdir, 'rootCA.pem'))
+
+        # attach policy to certificate
+        if self.options.policydocument is not '':
+            self.pd = ('{\n'
+                       '     "Version": "2012-10-17",\n'
+                       '     "Statement": [{\n'
+                       '     "Effect":	"Allow",\n'
+                       '       "Action":["iot:*"],\n'
+                       '       "Resource": ["*"]\n'
+                       '     }]\n'
+                       '}\n'
+                       )
+            self.options.policydocument = "policydocument"
+            with open(self.options.policydocument, "w") as text_file:
+                text_file.write(self.pd)
+
+        arnresp = os.path.join(self.builddir, 'arnresponse.txt')
+        try:
+            self.run_to_file(self.aws + ['create-policy',
+                                         '--policy-name',
+                                         self.options.policyname,
+                                         '--policy-document',
+                                         'file://' +
+                                         self.options.policydocument],
+                             arnresp)
+        except subprocess.CalledProcessError:
+            print("If the policy name already exists ' \
+                  'then creating it will fail. You can ignore this error.")
+            self.run_to_file(self.aws + ['get-policy',
+                                         '--policy-name',
+                                         self.options.policyname],
+                             arnresp)
+
+        with open('arnresponse.txt') as data_file:
+                self.data = json.load(data_file)
+
+        self.run(self.aws + ['attach-principal-policy',
+                             '-‐principal-arn',
+                             self.data["policyArn"],
+                             '--policy-name',
+                             self.options.policyname])
+
+        self.run(self.aws + ['create-thing',
+                             '--thing-name',
+                             self.options.thing])
+
+        print("Created Thing: %s" % self.options.thing)
+
+    def stage_fileset(self):
+        fileset = super().stage_fileset()
+        fileset.append('-certresponse.txt')
+        fileset.append('-arnresponse.txt')
+        return fileset

=== modified file 'snapcraft/plugins/python2.py'
--- snapcraft/plugins/python2.py	2015-10-27 19:23:02 +0000
+++ snapcraft/plugins/python2.py	2015-10-29 14:24:50 +0000
@@ -48,6 +48,15 @@
         schema['properties']['requirements'] = {
             'type': 'string',
         }
+        schema['properties']['pip-packages'] = {
+            'type': 'array',
+            'minitems': 1,
+            'uniqueItems': True,
+            'items': {
+                'type': 'string'
+            },
+            'default': [],
+        }
         schema.pop('required')
 
         return schema
@@ -76,7 +85,8 @@
         if self.options.requirements:
             requirements = os.path.join(os.getcwd(), self.options.requirements)
 
-        if not os.path.exists(setup) and not self.options.requirements:
+        if not os.path.exists(setup) and not \
+                (self.options.requirements or self.options.pip_packages):
             return
 
         easy_install = os.path.join(
@@ -100,6 +110,9 @@
         if self.options.requirements:
             self.run(pip_install + ['--requirement', requirements])
 
+        if self.options.pip_packages:
+            self.run(pip_install + ['--upgrade'] + self.options.pip_packages)
+
         if os.path.exists(setup):
             self.run(pip_install + ['.', ])
 

=== modified file 'snapcraft/plugins/python3.py'
--- snapcraft/plugins/python3.py	2015-10-27 19:23:02 +0000
+++ snapcraft/plugins/python3.py	2015-10-29 14:24:50 +0000
@@ -48,6 +48,15 @@
         schema['properties']['requirements'] = {
             'type': 'string',
         }
+        schema['properties']['pip-packages'] = {
+            'type': 'array',
+            'minitems': 1,
+            'uniqueItems': True,
+            'items': {
+                'type': 'string'
+            },
+            'default': [],
+        }
         schema.pop('required')
 
         return schema
@@ -76,7 +85,8 @@
         if self.options.requirements:
             requirements = os.path.join(os.getcwd(), self.options.requirements)
 
-        if not os.path.exists(setup) and not self.options.requirements:
+        if not os.path.exists(setup) and not \
+                (self.options.requirements or self.options.pip_packages):
             return
 
         easy_install = os.path.join(
@@ -99,6 +109,9 @@
         if self.options.requirements:
             self.run(pip_install + ['--requirement', requirements])
 
+        if self.options.pip_packages:
+            self.run(pip_install + ['--upgrade'] + self.options.pip_packages)
+
         if os.path.exists(setup):
             self.run(pip_install + ['.', ])
 

=== modified file 'snapcraft/tests/test_cmds.py'
--- snapcraft/tests/test_cmds.py	2015-10-26 15:54:30 +0000
+++ snapcraft/tests/test_cmds.py	2015-10-29 14:24:50 +0000
@@ -25,6 +25,7 @@
 from snapcraft import (
     cmds,
     common,
+    lifecycle,
     tests
 )
 
@@ -39,16 +40,12 @@
         self.addCleanup(tmpdirObject.cleanup)
         tmpdir = tmpdirObject.name
 
-        part1 = mock.Mock()
-        part1.name = 'part1'
-        part1.code.options.stage = ['*']
+        part1 = lifecycle.load_plugin('part1', 'jdk', {'source': '.'})
         part1.installdir = tmpdir + '/install1'
         os.makedirs(part1.installdir + '/a')
         open(part1.installdir + '/a/1', mode='w').close()
 
-        part2 = mock.Mock()
-        part2.name = 'part2'
-        part2.code.options.stage = ['*']
+        part2 = lifecycle.load_plugin('part2', 'jdk', {'source': '.'})
         part2.installdir = tmpdir + '/install2'
         os.makedirs(part2.installdir + '/a')
         with open(part2.installdir + '/1', mode='w') as f:
@@ -57,9 +54,7 @@
         with open(part2.installdir + '/a/2', mode='w') as f:
             f.write('a/2')
 
-        part3 = mock.Mock()
-        part3.name = 'part3'
-        part3.code.options.stage = ['*']
+        part3 = lifecycle.load_plugin('part3', 'jdk', {'source': '.'})
         part3.installdir = tmpdir + '/install3'
         os.makedirs(part3.installdir + '/a')
         os.makedirs(part3.installdir + '/b')
@@ -84,6 +79,8 @@
     def test_list_plugins(self, mock_stdout):
         expected_list = '''ant
 autotools
+awscli
+awsiot
 catkin
 cmake
 copy

=== modified file 'snapcraft/tests/test_lifecycle.py'
--- snapcraft/tests/test_lifecycle.py	2015-10-21 20:03:15 +0000
+++ snapcraft/tests/test_lifecycle.py	2015-10-29 14:24:50 +0000
@@ -165,7 +165,7 @@
                 dstdir = tmpdir + '/stage'
                 os.makedirs(dstdir)
 
-                files, dirs = snapcraft.lifecycle.migratable_filesets(
+                files, dirs = snapcraft.lifecycle._migratable_filesets(
                     filesets[key]['fileset'], srcdir)
                 snapcraft.lifecycle._migrate_files(files, dirs, srcdir, dstdir)