Merge lp:~noise/tanuki-agent/provkit-compose into lp:tanuki-agent

[Bret Barker]

1st pass - new provision2 script that knows how to u-d-f images from base_snaps+channel

This will likely require more work after other branches land, but so far will successfully read test opps that have additional base image data, e.g.:
    "base_snaps": [
        { "name": "ubuntu-core", "sequence": 7},
        { "name": "pi2.canonical", "sequence": 3},
        { "name": "test-snap.noise", "sequence": 1}
    ],
    "base_channel": "edge",
    "base_image_id": "qwerty12345",

check CPI to confirm sequence #s are current in base_channel, download the snaps, and compose an image with ubuntu-device-flash.

[Roberto Alsina (ralsina)]

+1

review: Approve

=== modified file 'Makefile'
--- Makefile	2015-10-30 21:14:35 +0000
+++ Makefile	2015-11-23 20:28:24 +0000
@@ -19,7 +19,7 @@
 
 raw-lint:
 	env/bin/flake8 *.py scripts/*.py --max-line-length=99
-	env/bin/flake8 rpi2-sample-provkit/setup rpi2-sample-provkit/provision rpi2-sample-provkit/runtest rpi2-sample-provkit/sshrobot --max-line-length=99 --builtins=raw_input
+	env/bin/flake8 rpi2-sample-provkit/setup rpi2-sample-provkit/provision rpi2-sample-provkit/provision2 rpi2-sample-provkit/runtest rpi2-sample-provkit/sshrobot rpi2-sample-provkit/utils.py --max-line-length=99 --builtins=raw_input
 	env/bin/flake8 rpi2-sample-provkit/test_tarball_content/test --max-line-length=99
 
 raw-test:
 
=== added file 'rpi2-sample-provkit/provision2'
--- rpi2-sample-provkit/provision2	1970-01-01 00:00:00 +0000
+++ rpi2-sample-provkit/provision2	2015-11-23 20:28:24 +0000
@@ -0,0 +1,114 @@
+#!/usr/bin/python -u
+
+from __future__ import print_function
+
+import argparse
+import codecs
+import json
+import os
+import subprocess
+import time
+
+import utils
+
+MAX_RETRIES = 5
+
+STEPS = """
+1. Write the image into the Raspberry Pi 2 SDCard (remote flashing of RPi2 is
+   still in-development), in another shell/window:
+
+     sudo dd if={} of=SDCARD_DEVICE bs=32M
+
+   where SDCARD_DEVICE
+   is the device in your system that holds the SDCard (like '/dev/sdb',
+   but be SURE that you're choosing the right device, otherwise you may
+   overwrite preexisting data on a device in your machine).
+
+2. Insert the card into the Raspberry Pi 2, and reboot it.
+"""
+
+log = utils.get_prov_kit_logger()
+
+
+def run_sshrobot(sshopts, cmd):
+    """Run a command using sshrobot helper."""
+    sshrobot = os.path.join(os.path.dirname(__file__), 'sshrobot')
+    invoke = [sshrobot] + sshopts.ssh + [cmd]
+    p = subprocess.Popen(invoke, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    buf = ""
+    while True:
+        r = p.poll()
+        if r is not None:
+            out = (buf + p.stdout.read().decode("utf8")).strip()
+            if out:
+                print(out)
+            return r
+        ch = p.stdout.read(1).decode("utf8")
+        buf += ch
+        if buf.endswith(" password: "):
+            break
+    p.stdin.write(sshopts.password + "\n")
+    p.stdin.flush()
+    r = p.wait()
+    out = p.stdout.read().decode("utf8").strip()
+    if out:
+        print(out)
+    return r
+
+
+def wait_device(sshopts):
+    """Wait for device to answer SSH."""
+    ssh_identity = os.getenv('SSH_IDENTITY', '~/.ssh/id_rsa.pub')
+    pubpath = os.path.expanduser(ssh_identity)
+    with open(pubpath, "rb") as fh:
+        pubcontent = fh.read()
+    cmd = "echo '" + pubcontent + b"' >> ~/.ssh/authorized_keys"
+    for attempt_number in range(1, MAX_RETRIES + 1):
+        res = run_sshrobot(sshopts, cmd)
+        if res == 0:
+            return
+        if attempt_number == MAX_RETRIES:
+            break
+        log.info("device still not answering, waiting 10s and retrying {}/{}..."
+                 .format(attempt_number, MAX_RETRIES-1))
+        time.sleep(10)
+
+    # we never reached the raspi in all attempts
+    log.error("device never answered, quitting!")
+    log.error("Please be sure that the Raspberry Pi 2 is in a functional state and retry.")
+    exit(1)
+
+
+def main(args):
+    """Main entry point."""
+    sshopts = utils.device_sshopts()
+    # get the test opportunity content
+    with codecs.open(args.test_opp, 'rt', encoding='utf8') as fh:
+        test_opp = json.load(fh)
+
+    log.info("Starting to provision the Raspberry Pi 2 (@ {address}:{port}),"
+             " for the test {spec_name}".format(address=sshopts.ip, port=sshopts.port, **test_opp))
+
+    # TODO: test_opp needs base_channel, base_snaps, architecture, base_image_id
+    arch = test_opp['platform']  # XXX: s/platform/architecture
+    release = test_opp['release']
+    channel = test_opp['base_channel']
+    base_snaps = test_opp['base_snaps']
+
+    image_filename = 'gm_{}_{}.img'.format(channel, test_opp['base_image_id'])
+    utils.compose_image(arch, release, channel, base_snaps, image_filename)
+
+    print(STEPS.format(image_filename, **test_opp))
+    print("Please press ENTER when all steps are done")
+    raw_input()
+
+    # wait for ssh to actually work
+    wait_device(sshopts)
+    log.info("Done.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("test_opp")
+    args = parser.parse_args()
+    main(args)
 
=== modified file 'rpi2-sample-provkit/setup'
--- rpi2-sample-provkit/setup	2015-10-30 21:14:35 +0000
+++ rpi2-sample-provkit/setup	2015-11-23 20:28:24 +0000
@@ -60,6 +60,7 @@
 
     # try to update in the running image
     for snap_name, snap_revno in to_update:
+        # XXX: use new CPI and download function in utils.py
         log.info('requesting package info for snap: %s, headers: %r', snap_name, headers)
         res = requests.get(PACKAGE_URL.format(snap_name), headers=headers)
         if res.status_code != 200:
 
=== modified file 'rpi2-sample-provkit/utils.py'
--- rpi2-sample-provkit/utils.py	2015-10-30 21:14:35 +0000
+++ rpi2-sample-provkit/utils.py	2015-11-23 20:28:24 +0000
@@ -1,15 +1,24 @@
 import logging
 import os
 import sys
+import requests
+import subprocess
+
+
+PACKAGE_URL = "https://search.apps.ubuntu.com/api/v1/package/{}"
 
 
 def get_prov_kit_logger():
     """Configure logging and return the provkit logger."""
     logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s: %(message)s",
                         stream=sys.stderr, level=logging.INFO)
+    logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
     return logging.getLogger("provkit.{}".format(os.path.basename(sys.argv[0])))
 
 
+log = get_prov_kit_logger()
+
+
 def device_sshopts():
     opts = ['-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null']
 
@@ -21,3 +30,84 @@
         ssh = opts + ['-q', '-p', str(port), target]
         scp = opts + ['-q', '-P', str(port)]
     return sshopts
+
+
+def get_snap_info(arch, release, channel, snap_name, snap_seq, bail_on_seq_mismatch=True):
+    """ Fetch package info from Click Package Index (CPI) and return the download URL
+    and a boolean is_oem if this is an OEM type package. """
+
+    headers = {
+        'X-Ubuntu-Architecture': arch,
+        'X-Ubuntu-Release': release
+    }
+
+    url = PACKAGE_URL.format(snap_name)
+    log.info('Requesting package info for snap: %s, url: %s, headers: %r', snap_name, url, headers)
+
+    res = requests.get(url, headers=headers)
+    if res.status_code != 200:
+        log.error('Cannot find package info for snap: %s, status_code: %d',
+                  snap_name, res.status_code)
+        sys.exit(1)
+    info = res.json()
+    if info['sequence'] != snap_seq:
+        log.error("cannot get hold of {} sequence {}, it's not the current latest: {}"
+                  .format(snap_name, snap_seq, info['sequence']))
+        sys.exit(1)
+
+    url = info['anon_download_url']
+    is_oem = info['content'] == 'oem'
+    return url, is_oem
+
+
+def download_snap(url, output_filename):
+    # TODO: consider keeping a cache of downloads
+    log.info("Downloading: {}".format(url))
+    subprocess.check_call(['curl', '-o', output_filename, url])
+
+
+def compose_image(arch, release, channel, base_snaps, image_filename):
+    """ Gather snap info, downlaod snaps, and use ubuntu-device-flash to compose an image"""
+    # gather snap info
+    oem = None
+    installs = []
+    for s in base_snaps:
+        if s['name'] == 'ubuntu-core':  # XXX: remove once we have all-snaps
+            log.warn('Skipping ubuntu-core snap because we are still in hybrid system-image world')
+            core = s
+            continue
+        s['url'], s['is_oem'] = get_snap_info(arch, release, channel, s['name'], s['sequence'])
+        s['fname'] = '{}_s{}.snap'.format(s['name'], s['sequence'])
+        if s['is_oem']:
+            oem = s['fname']
+        else:
+            installs += ['--install', s['fname']]
+
+    if core is not None:
+        base_snaps.remove(core)
+    if oem is None:
+        log.error('OEM snap is required to be specified in the Product, but none found')
+        sys.exit(1)
+    # log.info('base_snaps: %r', base_snaps)
+    log.info('oem: %s, installs: %r', oem, installs)
+
+    # downlaod snaps - TODO: parallelize this
+    for s in base_snaps:
+        download_snap(s['url'], s['fname'])
+
+    # TODO: consider writing the udf cmd or the list of snaps/seqs out next to the image
+
+    sys_image_release = 'rolling'  # XXX: ditch when all-snaps
+    sys_image_chan = 'edge'  # XXX: ditch when all-snaps
+    udf_args = ['sudo', 'ubuntu-device-flash', 'core', sys_image_release,
+                '--developer-mode',
+                '--device', 'raspi2_armhf',
+                '-o', image_filename,
+                '--channel', sys_image_chan,
+                '--oem', oem
+                ] + installs
+
+    log.info('Composing image with: %s', ' '.join(udf_args))
+    log.warn('Note: you may be required to enter your password for sudo. This is a result of the '
+             'current ubuntu-device-flash requiring root access.')
+    subprocess.check_call(udf_args)