Merge lp:~sinzui/juju-ci-tools/juju-core2 into lp:juju-ci-tools

[Curtis Hovey]

support juju-core and juju-2.0 binary package names.

This branch was started many months ago. It does not work since scripts and the packaging changed

Unmerged revisions

1293. By Curtis Hovey on 2016-11-01

Merged trunk

1292. By Curtis Hovey on 2016-06-27

Merged trunk

1291. By Curtis Hovey on 2016-03-02

Remove ambiguous uses og juju-core.

1290. By Curtis Hovey on 2016-03-02

Update s3ci to get juju2 package name.

1289. By Curtis Hovey on 2016-03-02

Removed unused certify scripts. Determine deb package name based on Juju version.

1288. By Curtis Hovey on 2016-03-02

Merged trunk.

=== modified file 'README.md'
--- README.md	2016-06-02 21:48:54 +0000
+++ README.md	2016-11-01 14:01:39 +0000
@@ -14,7 +14,7 @@
 making a tree of tools and metadata, and lastly publishing the tools.
 you can skip the tarball and package steps if you just want to publish
 the juju tools (AKA jujud, servers, agents). If you want to test a fix
-that is in the juju-core trunk branch, you can make your own release
+that is in the juju master branch, you can make your own release
 tarball and package.
 
 Once juju is published to the test clouds, individual tests can be performed.
 
=== modified file 'build_vagrant_boxes.py'
--- build_vagrant_boxes.py	2016-02-19 06:41:06 +0000
+++ build_vagrant_boxes.py	2016-11-01 14:01:39 +0000
@@ -23,8 +23,7 @@
 """
 
 SERIES_TO_NUMBERS = {
-    'trusty': '14.04',
-    'precise': '12.04',
+    'xenial': '16.04',
 }
 JENKINS_KVM = 'JENKINS_KVM'
 WORKSPACE = 'WORKSPACE'
@@ -33,8 +32,7 @@
 def get_package_globs(series, arch):
     series_number = SERIES_TO_NUMBERS[series]
     return {
-        'core': 'juju-core_*%s*_%s.deb' % (series_number, arch),
-        'local': 'juju-local_*%s*_all.deb' % series_number,
+        'juju2': 'juju2_*%s*_%s.deb' % (series_number, arch),
     }
 
 
@@ -78,8 +76,7 @@
     env = os.environ.copy()
     env['BUILD_FOR'] = '%s:%s' % (series, arch)
     if package_info is not None:
-        env['JUJU_CORE_PKG'] = package_info['core']
-        env['JUJU_LOCAL_PKG'] = package_info['local']
+        env['JUJU_CORE_PKG'] = package_info['juju2']
     builder_path = os.path.join(jenkins_kvm, 'build-juju-local.sh')
     build_dir = '%s-%s' % (series, arch)
     logging.info('Building Vagrant box for %s %s' % (series, arch))
@@ -140,11 +137,8 @@
         package_info = get_debian_packages(
             credentials, args.workspace, args.series, args.arch,
             args.use_ci_juju_packages)
-        if 'core' not in package_info:
-            logging.error('Could not find juju-core package')
-            sys.exit(1)
-        if 'local' not in package_info:
-            logging.error('Could not find juju-local package')
+        if 'juju2' not in package_info:
+            logging.error('Could not find juju2 package')
             sys.exit(1)
     else:
         package_info = None
 
=== modified file 'candidate.py'
--- candidate.py	2016-08-04 16:38:06 +0000
+++ candidate.py	2016-11-01 14:01:39 +0000
@@ -19,6 +19,7 @@
     get_artifacts,
     get_credentials,
     JENKINS_URL,
+    PackageNamer,
     PUBLISH_REVISION
 )
 from utility import (
@@ -80,7 +81,7 @@
     if not pr_number:
         pr_number = find_publish_revision_number(credentials, br_number)
     get_artifacts(
-        credentials, PUBLISH_REVISION, pr_number, 'juju-core*', candidate_dir,
+        credentials, PUBLISH_REVISION, pr_number, 'juju*deb', candidate_dir,
         dry_run=dry_run, verbose=verbose)
 
 
@@ -95,11 +96,12 @@
 
 
 def get_package(artifacts_path, version):
-    """Return the path to the expected juju-core package for the localhost."""
+    """Return the path to the expected juju package for the localhost."""
     release = subprocess.check_output(['lsb_release', '-sr']).strip()
     arch = get_deb_arch()
-    package_name = 'juju-core_{}-0ubuntu1~{}.1~juju1_{}.deb'.format(
-        version, release, arch)
+    deb_name = PackageNamer.get_juju_package_name(version)
+    package_name = '{}_{}-0ubuntu1~{}.1~juju1_{}.deb'.format(
+        deb_name, version, release, arch)
     package_path = os.path.join(artifacts_path, package_name)
     return package_path
 
 
=== modified file 'jujuci.py'
--- jujuci.py	2016-07-01 11:05:05 +0000
+++ jujuci.py	2016-11-01 14:01:39 +0000
@@ -40,7 +40,6 @@
 JENKINS_URL = 'http://juju-ci.vapour.ws:8080'
 BUILD_REVISION = 'build-revision'
 PUBLISH_REVISION = 'publish-revision'
-CERTIFY_UBUNTU_PACKAGES = 'certify-ubuntu-packages'
 
 Artifact = namedtuple('Artifact', ['file_name', 'location'])
 
@@ -174,13 +173,6 @@
     return get_juju_binary(credentials, file_name, build_data, workspace)
 
 
-def get_certification_bin(credentials, version, workspace):
-    build_data = get_build_data(JENKINS_URL, credentials,
-                                CERTIFY_UBUNTU_PACKAGES, 'lastBuild')
-    file_name = PackageNamer.factory().get_certification_package(version)
-    return get_juju_binary(credentials, file_name, build_data, workspace)
-
-
 def get_juju_binary(credentials, file_name, build_data, workspace):
     artifact = get_filename_artifact(file_name, build_data)
     target_path = os.path.join(workspace, artifact.file_name)
@@ -340,15 +332,6 @@
     parser_get_juju_bin.add_argument('workspace', nargs='?', default='.',
                                      help='The place to store binaries.')
     add_credential_args(parser_get_juju_bin)
-    parser_get_certification_bin = subparsers.add_parser(
-        'get-certification-bin',
-        help='Retrieve and extract juju binaries for certification.')
-    parser_get_certification_bin.add_argument(
-        'version', help='The version to get certification for.')
-    parser_get_certification_bin.add_argument(
-        'workspace', nargs='?', default='.',
-        help='The place to store binaries.')
-    add_credential_args(parser_get_certification_bin)
     parser_get_buildvars = subparsers.add_parser(
         'get-build-vars',
         help='Retrieve the build-vars for a build-revision.')
@@ -424,20 +407,22 @@
 class PackageNamer(Namer):
     """A class knows the names of packages."""
 
+    @staticmethod
+    def get_juju_package_name(version):
+        """Return the name of the package based on the Juju version."""
+        if version.startswith('1.'):
+            return 'juju-core'
+        return 'juju2'
+
     def get_release_package_suffix(self):
         return '-0ubuntu1~{distro_release}.1~juju1_{arch}.deb'.format(
             distro_release=self.distro_release, arch=self.arch)
 
     def get_release_package(self, version):
         return (
-            'juju-core_{version}{suffix}'
-            ).format(version=version, suffix=self.get_release_package_suffix())
-
-    def get_certification_package(self, version):
-        return (
-            'juju-core_{version}~{distro_release}.1_{arch}.deb'
-            ).format(version=version, distro_release=self.distro_release,
-                     arch=self.arch)
+            '{deb_name}_{version}{suffix}').format(
+                deb_name=self.get_juju_package_name(version),
+                version=version, suffix=self.get_release_package_suffix())
 
 
 class JobNamer(Namer):
@@ -471,10 +456,6 @@
                 dry_run=args.dry_run, verbose=args.verbose)
         elif args.command == 'get-juju-bin':
             print_now(get_juju_bin(credentials, args.workspace))
-        elif args.command == 'get-certification-bin':
-            path = get_certification_bin(credentials, args.version,
-                                         args.workspace)
-            print_now(path)
         elif args.command == 'get-build-vars':
             text = get_buildvars(
                 credentials, args.build, env=args.env,
 
=== modified file 'make-buildvars'
--- make-buildvars	2016-02-18 20:53:26 +0000
+++ make-buildvars	2016-11-01 14:01:39 +0000
@@ -41,7 +41,7 @@
 
 
 def get_git_revision_info(branch, revision_spec):
-    """Find the commit and juju-core version of a branch.
+    """Find the commit and juju version of a branch.
 
     Returns a tuple comparable to revno, revision-id, version.
     The revno is always None. The revision-id is the commit hash.
 
=== removed file 'retrieve-ubuntu-packages.bash'
--- retrieve-ubuntu-packages.bash	2015-04-06 20:11:19 +0000
+++ retrieve-ubuntu-packages.bash	1970-01-01 00:00:00 +0000
@@ -1,86 +0,0 @@
-#!/bin/bash
-# Download Ubuntu juju packages that match the version under test.
-set -eu
-
-: ${LOCAL_JENKINS_URL=http://juju-ci.vapour.ws:8080}
-ARTIFACTS_PATH=$WORKSPACE/artifacts
-
-: ${SCRIPTS=$(readlink -f $(dirname $0))}
-export PATH="$SCRIPTS:$PATH"
-
-UBUNTU_ARCH="http://archive.ubuntu.com/ubuntu/pool/universe/j/juju-core/"
-PORTS_ARCH="http://ports.ubuntu.com/pool/universe/j/juju-core/"
-ALL_ARCHIVES="$UBUNTU_ARCH $PORTS_ARCH"
-
-TRUSTY_AMD64="certify-trusty-amd64"
-TRUSTY_PPC64="certify-trusty-ppc64"
-TRUSTY_I386="certify-trusty-i386"
-
-set -x
-
-usage() {
-    echo "usage: $0 VERSION"
-    echo "  VERSION: The juju package version to retrive."
-    exit 1
-}
-
-
-check_deps() {
-    echo "Phase 0: Checking requirements."
-    has_deps=1
-    which lftp || has_deps=0
-    if [[ $has_deps == 0 ]]; then
-        echo "Install lftp."
-        exit 2
-    fi
-}
-
-
-retrieve_packages() {
-    # Retrieve the $RELEASE packages that contain jujud,
-    # or copy a locally built package.
-    echo "Phase 1: Retrieving juju-core packages from archives"
-    cd $WORKSPACE
-    for archive in $ALL_ARCHIVES; do
-        safe_archive=$(echo "$archive" | sed -e 's,//.*@,//,')
-        echo "checking $safe_archive for $VERSION."
-        lftp -c mirror -I "juju*${VERSION}*.deb" $archive;
-    done
-    if [ -d $WORKSPACE/juju-core ]; then
-        found=$(find $WORKSPACE/juju-core/ -name "*deb")
-        if [[ $found != "" ]]; then
-            mv $WORKSPACE/juju-core/*deb ./
-        fi
-        rm -r $WORKSPACE/juju-core
-    fi
-}
-
-
-start_series_arch_tests() {
-    [[ -z $TOKEN ]] && return 0
-    encoded_version=$(echo "$VERSION" | sed 's,[+],%2B,')
-    query="token=$TOKEN&VERSION=$encoded_version"
-    for job in $TRUSTY_AMD64 $TRUSTY_PPC64 $TRUSTY_I386; do
-        curl -o /dev/null \
-            "$LOCAL_JENKINS_URL/job/$job/buildWithParameters?$query"
-    done
-}
-
-
-TOKEN=""
-while [[ "${1-}" != "" && $1 =~ ^-.* ]]; do
-    case $1 in
-        --start-other-tests)
-            shift
-            TOKEN=$1
-            ;;
-    esac
-    shift
-done
-
-test $# -eq 1 || usage
-VERSION=$1
-
-check_deps
-retrieve_packages
-start_series_arch_tests
 
=== modified file 's3ci.py'
--- s3ci.py	2016-02-19 06:41:06 +0000
+++ s3ci.py	2016-11-01 14:01:39 +0000
@@ -99,7 +99,7 @@
     suffix = PackageNamer.factory().get_release_package_suffix()
     filtered = [
         (k, f) for k, f in filter_keys(keys, suffix)
-        if f.startswith('juju-core_')]
+        if f.startswith('juju2_') or f.startswith('juju-core_')]
     if len(filtered) == 0:
         raise PackageNotFound('Package could not be found.')
     return sorted(filtered, key=lambda x: int(
 
=== modified file 'tests/test_candidate.py'
--- tests/test_candidate.py	2016-08-04 16:38:06 +0000
+++ tests/test_candidate.py	2016-11-01 14:01:39 +0000
@@ -128,7 +128,7 @@
         self.assertEqual(options, kwargs)
         output, args, kwargs = ga_mock.mock_calls[1]
         self.assertEqual(
-            (credentials, 'publish-revision', 5678, 'juju-core*',
+            (credentials, 'publish-revision', 5678, 'juju*deb',
              '~/candidate/1.24.4-artifacts'),
             args)
         self.assertEqual(options, kwargs)
@@ -159,7 +159,26 @@
         self.assertEqual(
             ['1.21-artifacts', 'master-artifacts'], sorted(dirs))
 
-    def test_get_package(self):
+    def test_get_package_2x(self):
+        def subprocessor(*args, **kwargs):
+            command = args[0]
+            if 'lsb_release' in command:
+                return '14.04'
+            elif 'dpkg' in command:
+                return 'amd64'
+
+        with patch('subprocess.check_output',
+                   side_effect=subprocessor) as co_mock:
+            name = get_package('foo', '2.2.3')
+        self.assertEqual(
+            'foo/juju2_2.2.3-0ubuntu1~14.04.1~juju1_amd64.deb', name)
+        self.assertEqual(2, co_mock.call_count)
+        output, args, kwargs = co_mock.mock_calls[0]
+        self.assertEqual((['lsb_release', '-sr'], ), args)
+        output, args, kwargs = co_mock.mock_calls[1]
+        self.assertEqual((['dpkg', '--print-architecture'], ), args)
+
+    def test_get_package_1x(self):
         def subprocessor(*args, **kwargs):
             command = args[0]
             if 'lsb_release' in command:
 
=== modified file 'tests/test_jujuci.py'
--- tests/test_jujuci.py	2016-07-01 15:50:09 +0000
+++ tests/test_jujuci.py	2016-11-01 14:01:39 +0000
@@ -10,7 +10,6 @@
 from jujuci import (
     add_artifacts,
     Artifact,
-    CERTIFY_UBUNTU_PACKAGES,
     clean_environment,
     Credentials,
     CredentialsMissing,
@@ -18,7 +17,6 @@
     get_artifacts,
     get_build_data,
     get_buildvars,
-    get_certification_bin,
     get_credentials,
     get_job_data,
     get_juju_bin,
@@ -481,33 +479,6 @@
             JENKINS_URL, credentials, 'build-binary-wacky-a64', 'lastBuild')
         self.assertEqual(1, jnf_mock.call_count)
 
-    def test_get_certification_bin(self):
-        package_namer = PackageNamer('foo', 'bar', 'baz')
-        build_data = {
-            'url': 'http://foo/',
-            'artifacts': [{
-                'fileName': 'juju-core_36.1~0ubuntu1~bar.1_foo.deb',
-                'relativePath': 'baz',
-                }]
-            }
-        credentials = Credentials('jrandom', 'password1')
-
-        with self.get_juju_binary_mocks() as (workspace, ur_mock, cc_mock):
-            with patch('jujuci.get_build_data', return_value=build_data,
-                       autospec=True) as gbd_mock:
-                with patch.object(PackageNamer, 'factory',
-                                  return_value=package_namer):
-                    bin_loc = get_certification_bin(credentials,
-                                                    '36.1~0ubuntu1',
-                                                    workspace)
-        self.assertEqual(bin_loc, os.path.join(
-            workspace, 'extracted-bin', 'subdir', 'sub-subdir', 'juju'))
-        ur_mock.assert_called_once_with(
-            'http://jrandom:password1@foo/artifact/baz',
-            os.path.join(workspace, 'juju-core_36.1~0ubuntu1~bar.1_foo.deb'))
-        gbd_mock.assert_called_once_with(
-            JENKINS_URL, credentials, CERTIFY_UBUNTU_PACKAGES, 'lastBuild')
-
     @contextmanager
     def get_juju_binary_mocks(self):
         def mock_extract_deb(args):
@@ -749,16 +720,16 @@
 class TestPackageNamer(TestNamer):
 
     def test_get_release_package(self):
-        package_namer = PackageNamer('amd42', '42.34', 'wacky')
+        package_namer = PackageNamer('amd42', '18.04', 'wacky')
         self.assertEqual(
-            package_namer.get_release_package('27.6'),
-            'juju-core_27.6-0ubuntu1~42.34.1~juju1_amd42.deb')
+            package_namer.get_release_package('1.25'),
+            'juju-core_1.25-0ubuntu1~18.04.1~juju1_amd42.deb')
 
-    def test_get_certification_package(self):
-        package_namer = PackageNamer('amd42', '42.34', 'wacky')
-        self.assertEqual(
-            package_namer.get_certification_package('27.6~0ubuntu1'),
-            'juju-core_27.6~0ubuntu1~42.34.1_amd42.deb')
+    def test_get_juju_package_name(self):
+        self.assertEqual(
+            'juju-core', PackageNamer.get_juju_package_name('1.2.3'))
+        self.assertEqual(
+            'juju2', PackageNamer.get_juju_package_name('2.2.3'))
 
 
 class TestJobNamer(TestNamer):
 
=== modified file 'tests/test_s3ci.py'
--- tests/test_s3ci.py	2016-02-19 06:41:06 +0000
+++ tests/test_s3ci.py	2016-11-01 14:01:39 +0000
@@ -146,11 +146,12 @@
     return key
 
 
-def mock_package_key(revision_build, build=27, distro_release=None):
+def mock_package_key(revision_build, build=27,
+                     distro_release=None, version='1.24.0'):
     namer = PackageNamer.factory()
     if distro_release is not None:
         namer.distro_release = distro_release
-    package = namer.get_release_package('109.6')
+    package = namer.get_release_package(version)
     job = JobNamer.factory().get_build_binary_job()
     return mock_key(revision_build, job, build, package)
 
@@ -177,8 +178,18 @@
         use_context(self, patch('utility.get_deb_arch', return_value='amd65',
                                 autospec=True))
 
-    def test_find_package_key(self):
-        key = mock_package_key(390)
+    def test_find_package_key_1x(self):
+        key = mock_package_key(390, version='1.25.0')
+        bucket = mock_bucket([key])
+        namer = JobNamer.factory()
+        job = namer.get_build_binary_job()
+        found_key, filename = find_package_key(bucket, 390)
+        bucket.list.assert_called_once_with(get_job_path(390, job))
+        self.assertIs(key, found_key)
+        self.assertEqual(filename, get_key_filename(key))
+
+    def test_find_package_key_2x(self):
+        key = mock_package_key(390, version='2.1.0')
         bucket = mock_bucket([key])
         namer = JobNamer.factory()
         job = namer.get_build_binary_job()
@@ -201,7 +212,7 @@
             find_package_key(bucket, 390)
 
     def test_wrong_file(self):
-        key = mock_package_key(390)
+        key = mock_package_key(390, version='1.25.0')
         key.name = key.name.replace('juju-core', 'juju-dore')
         bucket = FakeBucket([key])
         with self.assertRaises(PackageNotFound):