Merge lp:~sinzui/juju-ci-tools/build-from-s3 into lp:juju-ci-tools

[Curtis Hovey]

Build juju agents from s3 artifacts.

Many of the canonical machines do not have access to the jenkins jobs to pickup assets to build. They do have access to s3. This branch switched build_juju.py to use s3ci.py instead of jujuci.py. These changes have been used in anger to build juju agents for s390x, ppc64el, arm64, and amd64.
    build-ubuntu-s390x-agent
    build-ubuntu-ppc64el-agent
    build-ubuntu-arm64-agent
    build-ubuntu-amd64-agent

build_juju.py now needs to know the s3 config to get the build-revision's juju tarfile. It also needs to know the architecture to pass to the crossbuild.py script. Only the "ubuntu-agent" subcommand takes the arch option, so we only append --goarch when needed.

This cannot merge without me also updating these jobs to pass in the s3 config.
    build-centos
    build-osx-client
    build-win-agent
    build-win-client

=== modified file 'build_juju.py'
--- build_juju.py	2016-12-01 15:34:05 +0000
+++ build_juju.py	2016-12-08 15:43:29 +0000
@@ -9,14 +9,13 @@
 import traceback
 
 from candidate import run_command
+from jujuconfig import get_juju_home
 from jujuci import (
     add_artifacts,
-    add_credential_args,
-    get_artifacts,
-    get_credentials,
     BUILD_REVISION,
     setup_workspace,
 )
+import s3ci
 
 
 DEFAULT_JUJU_RELEASE_TOOLS = os.path.realpath(
@@ -42,8 +41,17 @@
     return script
 
 
+def get_juju_tarfile(s3cfg_path, build, workspace_dir):
+    bucket = s3ci.get_qa_data_bucket(s3cfg_path)
+    files = s3ci.fetch_files(
+        bucket, build, BUILD_REVISION, 'juju-core_.*.tar.gz',
+        workspace_dir)
+    return files[0]
+
+
 def build_juju(credentials, product, workspace_dir, build,
-               juju_release_tools=None, dry_run=False, verbose=False):
+               juju_release_tools=None, goarch=None,
+               dry_run=False, verbose=False):
     """Build the juju product from a Juju CI build-revision in a workspace.
 
     The product is passed to juju-release-tools/crossbuild.py. The options
@@ -53,13 +61,12 @@
     to crossbuild.py.
     """
     setup_workspace(workspace_dir, dry_run=dry_run, verbose=verbose)
-    artifacts = get_artifacts(
-        credentials, BUILD_REVISION, build, 'juju-core_*.tar.gz',
-        workspace_dir, archive=False, dry_run=dry_run, verbose=verbose)
-    tar_artifact = artifacts[0]
+    tarfile = get_juju_tarfile(credentials, build, workspace_dir)
     crossbuild = get_crossbuild_script(juju_release_tools)
-    command = [
-        crossbuild, product, '-b', '~/crossbuild', tar_artifact.file_name]
+    command = [crossbuild, product, '-b', '~/crossbuild']
+    if product == 'ubuntu-agent' and goarch:
+        command.extend(['--goarch', goarch])
+    command.append(tarfile)
     run_command(command, dry_run=dry_run, verbose=verbose)
     add_artifacts(workspace_dir, ARTIFACT_GLOBS, dry_run=dry_run,
                   verbose=verbose)
@@ -68,6 +75,7 @@
 def parse_args(args=None):
     """Return the argument parser for this program."""
     parser = ArgumentParser("Build and package juju for an non-debian OS.")
+    default_config = os.path.join(get_juju_home(), 'juju-qa.s3cfg')
     parser.add_argument(
         '-d', '--dry-run', action='store_true', default=False,
         help='Do not make changes.')
@@ -82,23 +90,29 @@
         help='The path to the juju-release-tools dir, default: %s' %
               DEFAULT_JUJU_RELEASE_TOOLS)
     parser.add_argument(
+        '-c', '--config', default=default_config,
+        help=('s3cmd config file for credentials.  Default to '
+              'juju-qa.s3cfg in juju home.'))
+    parser.add_argument(
+        '-a', '--goarch', default=os.environ.get('GOARCH', None),
+        help=('The GOARCH. Defaults to GOARCH in the env.'))
+    parser.add_argument(
         'product', choices=['win-client', 'win-agent', 'osx-client', 'centos',
                             'ubuntu-agent'],
         help='the kind of juju to make and package.')
     parser.add_argument(
-        'workspace',  help='The path to the workspace to build in.')
-    add_credential_args(parser)
+        'workspace', help='The path to the workspace to build in.')
     parsed = parser.parse_args(args)
-    return parsed, get_credentials(parsed)
+    return parsed
 
 
 def main(argv):
     """Build and package juju for an non-debian OS."""
-    args, credentials = parse_args(argv)
+    args = parse_args(argv)
     try:
         build_juju(
-            credentials, args.product, args.workspace, args.build,
-            juju_release_tools=args.juju_release_tools,
+            args.config, args.product, args.workspace, args.build,
+            juju_release_tools=args.juju_release_tools, goarch=args.goarch,
             dry_run=args.dry_run, verbose=args.verbose)
     except Exception as e:
         print(e)

=== modified file 'tests/test_build_juju.py'
--- tests/test_build_juju.py	2016-12-01 15:34:05 +0000
+++ tests/test_build_juju.py	2016-12-08 15:43:29 +0000
@@ -1,15 +1,15 @@
-from mock import patch
+from mock import (
+    Mock,
+    patch,
+    )
 import os
 from unittest import TestCase
 
-from jujuci import (
-    Artifact,
-    Credentials,
-    )
 from build_juju import (
     ARTIFACT_GLOBS,
     build_juju,
     get_crossbuild_script,
+    get_juju_tarfile,
     main,
 )
 from utility import temp_dir
@@ -19,50 +19,80 @@
 
     def test_main_options(self):
         with patch('build_juju.build_juju', autospec=True) as mock:
-            main(['-d', '-v', '-b', '1234', 'win-client', './foo',
-                  '--user', 'jrandom', '--password', 'password1'])
-            args, kwargs = mock.call_args
-            self.assertEqual(
-                (Credentials('jrandom', 'password1'), 'win-client', './foo',
-                 '1234'), args)
-            self.assertTrue(kwargs['dry_run'])
-            self.assertTrue(kwargs['verbose'])
+            main(['-d', '-v', '-b', '1234', 'win-client', './foo'])
+            s3cfg = os.path.join(os.environ.get('JUJU_HOME'), 'juju-qa.s3cfg')
+            mock.assert_called_once_with(
+                s3cfg, 'win-client', './foo', '1234', goarch=None,
+                dry_run=True, juju_release_tools=None, verbose=True)
+
+    def test_main_options_with_arch(self):
+        with patch('build_juju.build_juju', autospec=True) as mock:
+            main(['-d', '-v', '-b', '1234', '-a', 's390x',
+                  'ubuntu-agent', './foo'])
+            s3cfg = os.path.join(os.environ.get('JUJU_HOME'), 'juju-qa.s3cfg')
+            mock.assert_called_once_with(
+                s3cfg, 'ubuntu-agent', './foo', '1234', goarch='s390x',
+                dry_run=True, juju_release_tools=None, verbose=True)
 
     def test_build_juju(self):
-        credentials = Credentials('jrandom', 'password1')
         with temp_dir() as base_dir:
             work_dir = os.path.join(base_dir, 'workspace')
             with patch('build_juju.setup_workspace', autospec=True) as sw_mock:
-                artifacts = [
-                    Artifact('juju-core_1.2.3.tar.gz', 'http:...')]
-                with patch('build_juju.get_artifacts',
-                           return_value=artifacts, autospec=True) as ga_mock:
+                with patch('build_juju.get_juju_tarfile',
+                           return_value='juju-core_1.2.3.tar.gz',
+                           autospec=True) as gt_mock:
                     with patch('build_juju.run_command') as rc_mock:
                         with patch('build_juju.add_artifacts', autospec=True
                                    ) as aa_mock:
                             build_juju(
-                                credentials, 'win-client', work_dir,
-                                'lastSucessful', dry_run=True, verbose=True)
-        self.assertEqual((work_dir, ), sw_mock.call_args[0])
-        self.assertEqual(
-            {'dry_run': True, 'verbose': True}, sw_mock.call_args[1])
-        self.assertEqual(
-            (credentials, 'build-revision', 'lastSucessful',
-             'juju-core_*.tar.gz', work_dir,),
-            ga_mock.call_args[0])
-        self.assertEqual(
-            {'archive': False, 'dry_run': True, 'verbose': True},
-            ga_mock.call_args[1])
-        crossbuild = get_crossbuild_script()
-        self.assertEqual(
-            ([crossbuild, 'win-client', '-b', '~/crossbuild',
-              'juju-core_1.2.3.tar.gz'], ),
-            rc_mock.call_args[0])
-        self.assertEqual(
-            {'dry_run': True, 'verbose': True}, rc_mock.call_args[1])
-        self.assertEqual((work_dir, ARTIFACT_GLOBS), aa_mock.call_args[0])
-        self.assertEqual(
-            {'dry_run': True, 'verbose': True}, aa_mock.call_args[1])
+                                's3cfg', 'win-client', work_dir, '1234',
+                                dry_run=True, verbose=True)
+        sw_mock.assert_called_once_with(
+            work_dir, dry_run=True, verbose=True)
+        gt_mock.assert_called_once_with('s3cfg', '1234', work_dir)
+        crossbuild = get_crossbuild_script()
+        rc_mock.assert_called_once_with(
+            [crossbuild, 'win-client', '-b', '~/crossbuild',
+             'juju-core_1.2.3.tar.gz'], dry_run=True, verbose=True)
+        aa_mock.assert_called_once_with(
+            work_dir, ARTIFACT_GLOBS, dry_run=True, verbose=True)
+
+    def test_build_juju_with_goarch_ubuntu_agent(self):
+        with temp_dir() as base_dir:
+            work_dir = os.path.join(base_dir, 'workspace')
+            with patch('build_juju.setup_workspace', autospec=True):
+                with patch('build_juju.get_juju_tarfile',
+                           return_value='juju-core_1.2.3.tar.gz',
+                           autospec=True):
+                    with patch('build_juju.run_command') as rc_mock:
+                        with patch('build_juju.add_artifacts', autospec=True):
+                            build_juju(
+                                's3cfg', 'ubuntu-agent', work_dir, '1234',
+                                goarch='s390x', dry_run=True, verbose=True)
+        crossbuild = get_crossbuild_script()
+        rc_mock.assert_called_once_with(
+            [crossbuild, 'ubuntu-agent', '-b', '~/crossbuild',
+             '--goarch', 's390x', 'juju-core_1.2.3.tar.gz'],
+            dry_run=True, verbose=True)
+
+    def test_build_juju_with_goarch_non_ubuntu_agent(self):
+        # GOARCH is never set when the product is not ubuntu-agent
+        with temp_dir() as base_dir:
+            work_dir = os.path.join(base_dir, 'workspace')
+            with patch('build_juju.setup_workspace', autospec=True):
+                with patch('build_juju.get_juju_tarfile',
+                           return_value='juju-core_1.2.3.tar.gz',
+                           autospec=True):
+                    with patch('build_juju.run_command') as rc_mock:
+                        with patch('build_juju.add_artifacts', autospec=True):
+                            build_juju(
+                                's3cfg', 'win-client', work_dir, '1234',
+                                goarch='s390x', dry_run=True, verbose=True)
+        crossbuild = get_crossbuild_script()
+        rc_mock.assert_called_once_with(
+            [crossbuild, 'win-client', '-b', '~/crossbuild',
+             'juju-core_1.2.3.tar.gz'],
+            dry_run=True, verbose=True)
 
     def test_get_crossbuild_script(self):
         self.assertEqual(
@@ -73,3 +103,16 @@
         self.assertEqual(
             os.path.join(parent_dir, 'juju-release-tools', 'crossbuild.py'),
             get_crossbuild_script())
+
+    def test_get_juju_tarfile(self):
+        s3cfg_path = './cloud-city/juju-qa.s3cfg'
+        bucket = Mock()
+        with patch('build_juju.s3ci.get_qa_data_bucket',
+                   return_value=bucket, autospec=True) as gb_mock:
+            with patch('build_juju.s3ci.fetch_files', autospec=True,
+                       return_value=['./juju-core_1.2.3.tar.gz']) as ff_mock:
+                tarfile = get_juju_tarfile(s3cfg_path, '1234', './')
+        self.assertEqual('./juju-core_1.2.3.tar.gz', tarfile)
+        gb_mock.assert_called_once_with('./cloud-city/juju-qa.s3cfg')
+        ff_mock.assert_called_once_with(
+            bucket, '1234', 'build-revision', 'juju-core_.*.tar.gz', './')