Merge lp:~abentley/juju-release-tools/generate-simplestreams into lp:juju-release-tools

[Aaron Bentley]

Additional scripts for generating parallel streams.

This branch provides additional scripts that are used for generating parallel streams.

[Curtis Hovey (sinzui)]

I have a comment, but it doesn't block this branch.

review: Approve (code)

=== added file 'agent_paths.py'
--- agent_paths.py	1970-01-01 00:00:00 +0000
+++ agent_paths.py	2015-10-09 18:05:56 +0000
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser
+import json
+import os.path
+import re
+import sys
+
+from generate_simplestreams import json_dump
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('input')
+    parser.add_argument('output')
+    args = parser.parse_args()
+    paths_hashes = {}
+    with open(args.input) as input_file:
+        stanzas = json.load(input_file)
+    hashes = {}
+    old_hash_urls = {}
+    for stanza in stanzas:
+        path_hash = stanza['sha256']
+        old_hash_urls[path_hash] = stanza['item_url']
+        path = os.path.join('agent', os.path.basename(stanza['path']))
+        path = re.sub('-win(2012(hv)?(r2)?|7|8|81)-', '-windows-', path)
+        paths_hashes.setdefault(path, stanza['sha256'])
+        if paths_hashes[path] != path_hash:
+            raise ValueError('Conflicting hash')
+        stanza['path'] = path
+        hashes[path] = path_hash
+    ph_list = {}
+    for path, path_hash in hashes.items():
+        ph_list.setdefault(path_hash, set()).add(path)
+    for path_hash, paths in ph_list.items():
+        if len(paths) > 1:
+            print(paths)
+    json_dump(stanzas, args.output)
+    agent_downloads = []
+    for stanza in stanzas:
+        agent_downloads.append({
+            'path': stanza['path'],
+            'sha256': stanza['sha256'],
+            'url': old_hash_urls[stanza['sha256']],
+        })
+    json_dump(agent_downloads, 'downloads-' + args.output)
+
+if __name__ == '__main__':
+    sys.exit(main())

=== added file 'copy-agents.bash'
--- copy-agents.bash	1970-01-01 00:00:00 +0000
+++ copy-agents.bash	2015-10-09 18:05:56 +0000
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eux
+source_streams=$1
+VERSION=$2
+agent_path=$3
+mkdir -p $agent_path
+for sourcepath in $(sstream-query $1/streams/v1/index.json version=$VERSION --output-format="%(item_url)s"); do
+    cp $sourcepath $agent_path/$(basename $sourcepath)
+done

=== added file 'download_agents.py'
--- download_agents.py	1970-01-01 00:00:00 +0000
+++ download_agents.py	2015-10-09 18:05:56 +0000
@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import json
+import os
+from urllib.request import urlopen
+import subprocess
+import sys
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('downloads_file', metavar='downloads-file')
+    args = parser.parse_args()
+    with open(args.downloads_file) as downloads_file:
+        downloads = json.load(downloads_file)
+    for download in downloads:
+        path = download['path']
+        if os.path.isfile(path):
+            print('File already exists: {}'.format(path))
+        else:
+            print('Downloading: {}'.format(path), end='')
+            sys.stdout.flush()
+            with open(download['path'], 'wb') as target:
+                with urlopen(download['url']) as source:
+                    while True:
+                        chunk = source.read(102400)
+                        if len(chunk) == 0:
+                            break
+                        target.write(chunk)
+                        print('.', end='', file=sys.stderr)
+                        sys.stderr.flush()
+            print('')
+        print('Verifying hash')
+        hashsum = subprocess.check_output(
+            ['sha256sum', path]).split(b' ', 1)[0]
+        hashsum = hashsum.decode('ascii')
+        expected = download['sha256']
+        print(' {}\n {}'.format(hashsum, expected))
+        if hashsum != expected:
+            raise ValueError('Incorrect hash for {}'.format(path))
+
+
+if __name__ == '__main__':
+    main()

=== added file 'make-stanzas.bash'
--- make-stanzas.bash	1970-01-01 00:00:00 +0000
+++ make-stanzas.bash	2015-10-09 18:05:56 +0000
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eux
+revision_build=$1
+VERSION=$2
+final_path=$3
+sstream-query $HOME/streams/juju-dist/testing/tools/streams/v1/index.json version=$VERSION > revision-build-$revision_build.repr
+ssquery_json.py revision-build-$revision_build.repr revision-build-$revision_build.json
+set_stream.py revision-build-$revision_build.json $final_path $revision_build --update-path

=== added file 'set_stream.py'
--- set_stream.py	1970-01-01 00:00:00 +0000
+++ set_stream.py	2015-10-09 18:05:56 +0000
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+from argparse import ArgumentParser
+import json
+import os
+
+from generate_simplestreams import json_dump
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument('in_file')
+    parser.add_argument('out_file')
+    parser.add_argument('revision_build')
+    parser.add_argument('--update-path', action='store_true')
+    args = parser.parse_args()
+    with open(args.in_file) as in_file:
+        stanzas = json.load(in_file)
+    stream = 'revision-build-{}'.format(args.revision_build)
+    for stanza in stanzas:
+        stanza['content_id'] = 'com.ubuntu.juju:{}:tools'.format(stream)
+        if not args.update_path:
+            continue
+        path = os.path.join(
+            'agent', 'revision-build-{}'.format(args.revision_build),
+            os.path.basename(stanza['path']))
+        stanza['path'] = path
+    json_dump(stanzas, args.out_file)
+
+
+if __name__ == '__main__':
+    main()

=== added file 'ssquery_json.py'
--- ssquery_json.py	1970-01-01 00:00:00 +0000
+++ ssquery_json.py	2015-10-09 18:05:56 +0000
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+from argparse import ArgumentParser
+
+from generate_simplestreams import json_dump
+
+
+def parse_args():
+    parser = ArgumentParser(description='Convert sstream-query output to JSON')
+    parser.add_argument('input')
+    parser.add_argument('output')
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    output = []
+    with open(args.input) as in_file:
+        for line in in_file:
+            output.append(eval(line))
+    json_dump(output, args.output)
+
+
+if __name__ == '__main__':
+    main()

=== added file 'validate-streams.bash'
--- validate-streams.bash	1970-01-01 00:00:00 +0000
+++ validate-streams.bash	2015-10-09 18:05:56 +0000
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -eux
+testing=$1
+sstream-query $testing/streams/v1/index2.json content_id="com.ubuntu.juju:revision-build-$revision_build:tools" version=$VERSION --output-format="%(sha256)s  %(item_url)s" > sha256sums
+sha256sum -c sha256sums