Merge lp:~sinzui/juju-release-tools/one-cloud-publish into lp:juju-release-tools

[Curtis Hovey]

Verify local metadata to cloud metadata.

This branch introduces a rewrite of the publish-public-tools.bash script. The publish_streams.py script can verify a local stream with a remote stream just like the older script. eg.

    ../juju-release-tools/publish_streams.py -d -v -r testing released ./juju-dist azure

My next branch will do syncing.

[Martin Packman (gz)]

Looks good, some comments inline.

review: Approve

[lp:~sinzui/juju-release-tools/one-cloud-publish]

217. By Curtis Hovey on 2015-08-17

simplify the CPCS dict.

218. By Curtis Hovey on 2015-08-17

Updated docstrings.

[Curtis Hovey (sinzui)]

Thank you for the review. I updated the code and commented inline regarding why "stream" is not used in the verification case.

=== added file 'publish_streams.py'
--- publish_streams.py	1970-01-01 00:00:00 +0000
+++ publish_streams.py	2015-08-17 17:00:00 +0000
@@ -0,0 +1,127 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+from argparse import ArgumentParser
+import difflib
+import os
+import sys
+import traceback
+import urllib2
+
+
+CPCS = {
+    'aws': "http://juju-dist.s3.amazonaws.com",
+    'azure': "https://jujutools.blob.core.windows.net/juju-tools",
+    'canonistack': (
+        "https://swift.canonistack.canonical.com"
+        "/v1/AUTH_526ad877f3e3464589dc1145dfeaac60/juju-dist"),
+    'hp': (
+        "https://region-a.geo-1.objects.hpcloudsvc.com"
+        "/v1/60502529753910/juju-dist"),
+    'joyent': (
+        "https://us-east.manta.joyent.com/cpcjoyentsupport/public/juju-dist"),
+    }
+
+
+def get_remote_file(url):
+    """Return the content of a remote file."""
+    response = urllib2.urlopen(url)
+    content = response.read()
+    return content
+
+
+def diff_files(local, remote):
+    """Return the difference of a local and a remote file.
+
+    :return: a tuple of identical (True, None) or different (False, str).
+    """
+    with open(local, 'r') as f:
+        local_lines = f.read().splitlines()
+    remote_lines = get_remote_file(remote).splitlines()
+    diff_gen = difflib.unified_diff(local_lines, remote_lines, local, remote)
+    diff = '\n'.join(list(diff_gen))
+    if diff:
+        return False, diff
+    return True, None
+
+
+def verify_metadata(location, remote_stream, verbose=False):
+    """Verify all the streams metadata in a cloud matches the local metadata.
+
+
+    This verifies all the streams, not a single stream, to ensure the cloud
+    has exactly the same metadata as the local instance.
+    """
+    local_metadata = os.path.join(location, 'tools', 'streams', 'v1')
+    remote_metadata = '{}/tools/streams/v1'.format(remote_stream)
+    if verbose:
+        print('comparing {} to {}'.format(local_metadata, remote_metadata))
+    for data_file in os.listdir(local_metadata):
+        if data_file.endswith('.json'):
+            local_file = os.path.join(local_metadata, data_file)
+            remote_file = '{}/{}'.format(remote_metadata, data_file)
+            if verbose:
+                print('comparing {}'.format(data_file))
+            identical, diff = diff_files(local_file, remote_file)
+            if not identical:
+                return False, diff
+    if verbose:
+        print('All json matches')
+    return True, None
+
+
+def publish(stream, location, cloud,
+            remote_root=None, dry_run=False, verbose=False):
+    """Publish a stream to a cloud and verify it."""
+    if remote_root:
+        remote_stream = '{}/{}'.format(CPCS[cloud], remote_root)
+    else:
+        remote_stream = CPCS.get(cloud)
+    verify_metadata(location, remote_stream, verbose=verbose)
+
+
+def parse_args(argv=None):
+    """Return the argument parser for this program."""
+    parser = ArgumentParser("Publish streams to a cloud.")
+    parser.add_argument(
+        '-d', '--dry-run', action="store_true", help='Do not make changes.')
+    parser.add_argument(
+        '-v', '--verbose', action="store_true", help='Increase verbosity.')
+    parser.add_argument(
+        '-r', '--remote-root',
+        help='An alternate root to publish to such as testing or weekly.')
+    parser.add_argument(
+        'stream', help='The agent-stream to publish.',
+        choices=['released', 'proposed', 'devel'])
+    parser.add_argument(
+        'location', type=os.path.expanduser,
+        help='The path to the local tree of all streams (tools/).')
+    parser.add_argument(
+        'cloud', help='The destination cloud.',
+        choices=['streams', 'aws', 'azure', 'hp', 'joyent', 'canonistack'])
+    return parser.parse_args(argv)
+
+
+def main(argv):
+    """Publish streams to a cloud."""
+    args = parse_args(argv)
+    try:
+        publish(
+            args.stream, args.location, args.cloud,
+            remote_root=args.remote_root, dry_run=args.dry_run,
+            verbose=args.verbose)
+    except Exception as e:
+        print('{}: {}'.format(e.__class__.__name__, e))
+        if args.verbose:
+            traceback.print_tb(sys.exc_info()[2])
+        return 2
+    if args.verbose:
+        print("Done.")
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

=== added file 'tests/test_publish_streams.py'
--- tests/test_publish_streams.py	1970-01-01 00:00:00 +0000
+++ tests/test_publish_streams.py	2015-08-17 17:00:00 +0000
@@ -0,0 +1,127 @@
+from mock import patch
+import os
+import re
+from StringIO import StringIO
+from unittest import TestCase
+
+from publish_streams import (
+    CPCS,
+    diff_files,
+    get_remote_file,
+    main,
+    parse_args,
+    publish,
+    verify_metadata,
+)
+
+from utils import temp_dir
+
+
+class PublishStreamsTestCase(TestCase):
+
+    def test_parse_args(self):
+        args = parse_args(
+            ['-d', '-v', '-r', 'testing', 'released', '~/juju-dist', 'aws'])
+        self.assertTrue(args.dry_run)
+        self.assertTrue(args.verbose)
+        self.assertEqual('testing', args.remote_root)
+        self.assertEqual('released', args.stream)
+        self.assertEqual(os.path.expanduser('~/juju-dist'), args.location)
+        self.assertEqual('aws', args.cloud)
+
+    @patch('publish_streams.publish', autospec=True)
+    def test_main(self, p_mock):
+        exit_code = main(['-r', 'testing', 'released', '~/juju-dist', 'aws'])
+        self.assertEqual(0, exit_code)
+        p_mock.assert_called_with(
+            'released', os.path.expanduser('~/juju-dist'),
+            'aws', remote_root='testing', dry_run=False, verbose=False)
+
+    @patch('publish_streams.urllib2.urlopen', autospec=True)
+    def test_get_remote_file(self, uo_mock):
+        uo_mock.return_value = StringIO('data')
+        content = get_remote_file('http://foo/bar.json')
+        uo_mock.assert_called_with('http://foo/bar.json')
+        self.assertEqual('data', content)
+
+    @patch('publish_streams.get_remote_file', autospec=True)
+    def test_diff_files(self, gr_mock):
+        gr_mock.return_value = 'one\ntwo\nthree'
+        with temp_dir() as base:
+            local_path = os.path.join(base, 'bar.json')
+            with open(local_path, 'w') as local_file:
+                local_file.write('one\ntwo\nthree')
+            identical, diff = diff_files(local_path, 'http://foo/bar.json')
+        self.assertTrue(identical)
+        self.assertIsNone(diff)
+        gr_mock.assert_called_with('http://foo/bar.json')
+
+    @patch('publish_streams.get_remote_file', autospec=True)
+    def test_diff_files_different(self, gr_mock):
+        gr_mock.return_value = 'one\ntwo\nfour'
+        with temp_dir() as base:
+            local_path = os.path.join(base, 'bar.json')
+            with open(local_path, 'w') as local_file:
+                local_file.write('one\ntwo\nthree')
+            identical, diff = diff_files(local_path, 'http://foo/bar.json')
+        self.assertFalse(identical)
+        normalized_diff = re.sub('/tmp/.*/bar', '/tmp/bar', diff)
+        self.assertEqual(
+            '--- /tmp/bar.json\n\n'
+            '+++ http://foo/bar.json\n\n'
+            '@@ -1,3 +1,3 @@\n\n'
+            ' one\n'
+            ' two\n'
+            '-three\n'
+            '+four',
+            normalized_diff)
+
+    @patch('publish_streams.diff_files', autospec=True)
+    def test_verify_metadata(self, df_mock):
+        df_mock.return_value = (True, None)
+        with temp_dir() as base:
+            metadata_path = os.path.join(base, 'tools', 'streams', 'v1')
+            os.makedirs(metadata_path)
+            metadata_file = os.path.join(metadata_path, 'bar.json')
+            metadasta_sig = os.path.join(metadata_path, 'bar.json.sig')
+            with open(metadata_file, 'w') as local_file:
+                local_file.write('bar.json')
+            with open(metadasta_sig, 'w') as local_file:
+                local_file.write('bar.json.sig')
+            identical, diff = verify_metadata(base, CPCS['aws'], verbose=False)
+        self.assertTrue(identical)
+        self.assertIsNone(diff)
+        self.assertEqual(1, df_mock.call_count)
+        df_mock.assert_called_with(
+            metadata_file, '{}/tools/streams/v1/bar.json'.format(CPCS['aws']))
+
+    @patch('publish_streams.diff_files', autospec=True)
+    def test_verify_metadata_with_root_faile(self, df_mock):
+        df_mock.return_value = (False, 'different')
+        with temp_dir() as base:
+            metadata_path = os.path.join(base, 'tools', 'streams', 'v1')
+            os.makedirs(metadata_path)
+            metadata_file = os.path.join(metadata_path, 'bar.json')
+            with open(metadata_file, 'w') as local_file:
+                local_file.write('bar.json')
+            identical, diff = verify_metadata(base, CPCS['aws'])
+        self.assertFalse(identical)
+        self.assertEqual(diff, 'different')
+        df_mock.assert_called_with(
+            metadata_file, '{}/tools/streams/v1/bar.json'.format(CPCS['aws']))
+
+    @patch('publish_streams.verify_metadata', autospec=True)
+    def test_publish(self, vm_mock):
+        vm_mock.return_value = (True, None)
+        publish('testing', '/streams/juju-dist', 'aws',
+                remote_root=None, dry_run=False, verbose=False)
+        vm_mock.assert_called_with(
+            '/streams/juju-dist', CPCS['aws'], verbose=False)
+
+    @patch('publish_streams.verify_metadata', autospec=True)
+    def test_publish_with_remote(self, vm_mock):
+        vm_mock.return_value = (True, None)
+        publish('testing', '/streams/juju-dist', 'aws',
+                remote_root='weekly', dry_run=False, verbose=False)
+        vm_mock.assert_called_with(
+            '/streams/juju-dist', '%s/weekly' % CPCS['aws'], verbose=False)