Merge lp:~andrewjbeach/juju-release-tools/no-debs into lp:juju-release-tools

[Curtis Hovey]

Do not make agents from deb.

=== modified file 'generate_agents.py'
--- generate_agents.py	2016-12-02 17:00:50 +0000
+++ generate_agents.py	2017-01-25 14:33:53 +0000
@@ -4,87 +4,21 @@
 from argparse import (
     ArgumentParser,
     )
-from collections import namedtuple
 from datetime import datetime
 import errno
-import glob
 import os
-import re
 import shutil
 import subprocess
 import sys
-import tarfile
-from urlparse import (
-    urlsplit,
-    urlunsplit,
-    )
-
-from debian import deb822
 
 from agent_archive import parse_args as parse_archive_args
-from build_package import juju_series
 from make_agent_json import StanzaWriter
-from utils import temp_dir
-
-
-class NoDebsFound(Exception):
-    """Raised when no .deb files could be found."""
-
-
-# These are the archives that are searched for matching releases.
-UBUNTU_ARCH = "http://archive.ubuntu.com/ubuntu/pool/universe/j/juju-core/"
-ARM_ARCH = "http://ports.ubuntu.com/pool/universe/j/juju-core/"
-PUBLIC_ARCHIVES = [UBUNTU_ARCH, ARM_ARCH]
-
-
-def move_debs(dest_debs):
-    juju_core_dir = os.path.join(dest_debs, 'juju-2.0')
-    debs = glob.glob(os.path.join(juju_core_dir, '*deb'))
-    if len(debs) == 0:
-        # The juju-2.0 package was not found, try the juju-core package.
-        print('No debs in {}'.format(juju_core_dir))
-        juju_core_dir = os.path.join(dest_debs, 'juju-core')
-        debs = glob.glob(os.path.join(juju_core_dir, '*deb'))
-    if len(debs) == 0:
-        print('No debs in {}'.format(juju_core_dir))
-        raise NoDebsFound('No deb files found.')
-    for deb in debs:
-        shutil.move(deb, dest_debs)
-    shutil.rmtree(juju_core_dir)
-
-
-def retrieve_deb_packages(release, upatch, archives, dest_debs):
-    if '-' in release:
-        deb_release = release.replace('-', '~')
-    else:
-        deb_release = release
-    for archive in archives:
-        scheme, netloc, path, query, fragment = urlsplit(archive)
-        # Strip username / password
-        netloc = netloc.rsplit('@')[-1]
-        safe_archive = urlunsplit((scheme, netloc, path, query, fragment))
-        print("checking {} for {}".format(safe_archive, deb_release))
-        try:
-            subprocess.call([
-                'lftp', '-c', 'mirror', '-i',
-                "(juju-2.0|juju-core).*{}.*\.{}~juj.*\.deb$".format(
-                    deb_release, upatch),
-                archive], cwd=dest_debs)
-        except OSError as e:
-            if e.errno == errno.ENOENT:
-                print("%s not found in %s" % (deb_release, safe_archive))
-            else:
-                raise e
-        except subprocess.CalledProcessError:
-            raise
-
-
-def retrieve_packages(release, upatch, archives, dest_debs, s3_config):
+
+
+def retrieve_packages(release, dest_debs, s3_config):
     # Retrieve the packages that contain a jujud for this version.
     print("Retrieving juju-core packages from archives")
     print(datetime.now().replace(microsecond=0).isoformat())
-    retrieve_deb_packages(release, upatch, archives, dest_debs)
-    move_debs(dest_debs)
     if os.path.exists(s3_config):
         print(
             'checking s3://juju-qa-data/agent-archive for'
@@ -105,75 +39,9 @@
     parser.add_argument('release', help='The juju release to prepare')
     parser.add_argument('agent_stream', help='The juju agent-stream.')
     parser.add_argument('destination', help='The simplestreams destination')
-    parser.add_argument('--upatch', help='Ubuntu patchlevel', default='1')
     return parser.parse_args()
 
 
-def list_ppas(juju_home):
-    config = os.path.join(juju_home, 'buildarchrc')
-    if not os.path.exists(config):
-        return None
-    listing = subprocess.check_output(
-        ['/bin/bash', '-c', 'source {}; set -u; echo '
-         '"$BUILD_STABLE1_ARCH\n'
-         '$BUILD_DEVEL1_ARCH\n'
-         '$BUILD_SUPPORTED_ARCH"'.format(config)])
-    return listing.splitlines()
-
-
-AgentVersion = namedtuple(
-    'AgentVersion', ['version', 'major_minor', 'series', 'architecture'])
-
-
-def get_agent_version(control_str):
-    control = deb822.Deb822(control_str)
-    control_version = control['Version']
-    corrected_version = re.sub('~(alpha|beta|rc)', r'-\1', control_version)
-    epochless_version = re.sub('^.:', '', corrected_version)
-    version = re.sub('(~|-0ubuntu).*$', '', epochless_version)
-    major_minor = '.'.join(version.split('-')[0].split('.')[0:2])
-    series = juju_series.get_name_from_package_version(control_version)
-    architecture = control['Architecture']
-    return AgentVersion(version, major_minor, series, architecture)
-
-
-def deb_to_agent(deb_path, dest_dir, agent_stream):
-    control_str = subprocess.check_output(['dpkg-deb', '-I', deb_path,
-                                           'control'])
-    agent_version = get_agent_version(control_str)
-    with temp_dir() as work_dir:
-        contents = os.path.join(work_dir, 'contents')
-        os.mkdir(contents)
-        subprocess.check_call(['dpkg-deb', '-x', deb_path, contents])
-        jujud_path = os.path.join(
-            contents, 'usr', 'lib', 'juju-{}'.format(
-                agent_version.major_minor), 'bin',
-            'jujud')
-        if not os.path.exists(jujud_path):
-            jujud_path = os.path.join(
-                contents, 'usr', 'lib', 'juju-{}'.format(
-                    agent_version.version), 'bin', 'jujud')
-        basename = 'juju-{}-{}-{}.tgz'.format(
-            agent_version.version, agent_version.series,
-            agent_version.architecture)
-        agent_filename = os.path.join(work_dir, basename)
-        with tarfile.open(agent_filename, 'w:gz') as tf:
-            tf.add(jujud_path, 'jujud')
-        writer = StanzaWriter.for_ubuntu(
-            juju_series.get_version(agent_version.series),
-            agent_version.series, agent_version.architecture,
-            agent_version.version, agent_filename, agent_stream=agent_stream)
-        writer.write_stanzas()
-        shutil.move(writer.filename, dest_dir)
-        final_agent_path = os.path.join(dest_dir, writer.path)
-        move_create_parent(agent_filename, final_agent_path)
-
-
-def debs_to_agents(dest_debs, agent_stream):
-    for deb_path in glob.glob(os.path.join(dest_debs, '*.deb')):
-        deb_to_agent(deb_path, dest_debs, agent_stream)
-
-
 def move_create_parent(source, target):
     try:
         os.makedirs(os.path.dirname(target))
@@ -183,6 +51,19 @@
     shutil.move(source, target)
 
 
+def make_ubuntu_agent(dest_debs, agent_stream, release):
+    for arch in ['amd64', 'arm64', 'ppc64el', 's390x']:
+        tarfile = os.path.join(
+            dest_debs, 'juju-{}-ubuntu-{}.tgz'.format(release, arch))
+        writer = StanzaWriter.for_living_ubuntu(
+            arch, release, tarfile, agent_stream=agent_stream)
+        writer.write_stanzas()
+        agent_path = os.path.join(dest_debs, writer.path)
+        shutil.copy2(tarfile, agent_path)
+        arch_name = '{}-{}'.format(arch, writer.filename)
+        shutil.move(writer.filename, os.path.join(dest_debs, arch_name))
+
+
 def make_windows_agent(dest_debs, agent_stream, release):
     source = os.path.join(
         dest_debs, 'juju-{}-win2012-amd64.tgz'.format(release))
@@ -214,15 +95,8 @@
     juju_dir = os.environ.get(
         'JUJU_HOME', os.path.join(os.environ.get('HOME'), '.juju'))
     s3_config = os.path.join(juju_dir, 'juju-qa.s3cfg')
-    archives = list_ppas(juju_dir)
-    if archives is None:
-        print("Only public archives will be searched.")
-        archives = PUBLIC_ARCHIVES
-    else:
-        print("Searching the build archives.")
-    retrieve_packages(args.release, args.upatch, archives, dest_debs,
-                      s3_config)
-    debs_to_agents(dest_debs, args.agent_stream)
+    retrieve_packages(args.release, dest_debs, s3_config)
+    make_ubuntu_agent(dest_debs, args.agent_stream, args.release)
     make_windows_agent(dest_debs, args.agent_stream, args.release)
     make_centos_agent(dest_debs, args.agent_stream, args.release)
 

=== modified file 'tests/test_agent_archive.py'
--- tests/test_agent_archive.py	2016-12-02 17:00:50 +0000
+++ tests/test_agent_archive.py	2017-01-25 14:33:53 +0000
@@ -12,7 +12,6 @@
     is_new_version,
     listing_to_files,
     main,
-    parse_args,
     )
 
 from utils import temp_dir

=== modified file 'tests/test_generate_agents.py'
--- tests/test_generate_agents.py	2016-12-02 17:00:50 +0000
+++ tests/test_generate_agents.py	2017-01-25 14:33:53 +0000
@@ -2,97 +2,19 @@
 
 from contextlib import contextmanager
 import os
-from mock import (
-    call,
-    patch
-    )
 from unittest import TestCase
 
+from mock import patch
+
 from generate_agents import (
-    get_agent_version,
     make_centos_agent,
     make_windows_agent,
-    move_debs,
-    NoDebsFound,
-    retrieve_deb_packages,
+    make_ubuntu_agent,
     retrieve_packages,
     )
 from utils import temp_dir
 
 
-class TestMoveDebs(TestCase):
-
-    def test_juju_2(self):
-        with temp_dir() as dest_debs:
-            parent = os.path.join(dest_debs, 'juju-2.0')
-            os.mkdir(parent)
-            open(os.path.join(parent, 'foo.deb'), 'w').close()
-            move_debs(dest_debs)
-            self.assertTrue(os.path.exists(os.path.join(dest_debs, 'foo.deb')))
-
-    def test_juju_core(self):
-        with temp_dir() as dest_debs:
-            parent = os.path.join(dest_debs, 'juju-core')
-            os.mkdir(parent)
-            open(os.path.join(parent, 'foo.deb'), 'w').close()
-            move_debs(dest_debs)
-            self.assertTrue(os.path.exists(os.path.join(dest_debs, 'foo.deb')))
-
-    def test_none(self):
-        with temp_dir() as dest_debs:
-            parent = os.path.join(dest_debs, 'juju-core')
-            os.mkdir(parent)
-            with self.assertRaisesRegexp(NoDebsFound, 'No deb files found.'):
-                move_debs(dest_debs)
-
-    def test_wrong_dir(self):
-        with temp_dir() as dest_debs:
-            parent = os.path.join(dest_debs, 'wrong-dir')
-            os.mkdir(parent)
-            open(os.path.join(parent, 'foo.deb'), 'w').close()
-            with self.assertRaisesRegexp(NoDebsFound, 'No deb files found.'):
-                move_debs(dest_debs)
-
-
-class TestRetreiveDebPackages(TestCase):
-
-    archives = [
-        'https://me:[e-mail]/team/stable',
-        'https://me:[e-mail]/team/devel',
-        ]
-
-    def test_retrieve_deb_packages_stable(self):
-        with patch('subprocess.call', autospec=True) as c_mock:
-            retrieve_deb_packages('2.1.0', '1', self.archives, '2.1.0/')
-        calls = [
-            call(['lftp', '-c', 'mirror', '-i',
-                  '(juju-2.0|juju-core).*2.1.0.*\\.1~juj.*\\.deb$',
-                  'https://me:[e-mail]/team/stable'],
-                 cwd='2.1.0/'),
-            call(['lftp', '-c', 'mirror', '-i',
-                  '(juju-2.0|juju-core).*2.1.0.*\\.1~juj.*\\.deb$',
-                  'https://me:[e-mail]/team/devel'],
-                 cwd='2.1.0/')
-            ]
-        self.assertEqual(calls, c_mock.mock_calls)
-
-    def test_retrieve_deb_packages_devel(self):
-        with patch('subprocess.call', autospec=True) as c_mock:
-            retrieve_deb_packages(
-                '2.1-beta1', '1', self.archives, '2.1-beta1/')
-        calls = [
-            call(['lftp', '-c', 'mirror', '-i',
-                  '(juju-2.0|juju-core).*2.1~beta1.*\\.1~juj.*\\.deb$',
-                  'https://me:[e-mail]/team/stable'],
-                 cwd='2.1-beta1/'),
-            call(['lftp', '-c', 'mirror', '-i',
-                  '(juju-2.0|juju-core).*2.1~beta1.*\\.1~juj.*\\.deb$',
-                  'https://me:[e-mail]/team/devel'],
-                 cwd='2.1-beta1/')
-            ]
-        self.assertEqual(calls, c_mock.mock_calls)
-
-
 control_string_template = """\
 Package: juju-2.0
 Source: juju-core
@@ -151,25 +73,26 @@
         self.assertEqual(args.config, s3_config)
 
 
-class TestGetAgentVersion(TestCase):
-
-    def test_get_agent_version_stable(self):
-        control_str = control_string_template.format(
-            '1:2.1.0-0ubuntu1~16.04.1~juju1')
-        agent_version = get_agent_version(control_str)
-        self.assertEqual('2.1.0', agent_version.version)
-        self.assertEqual('2.1', agent_version.major_minor)
-        self.assertEqual('xenial', agent_version.series)
-        self.assertEqual('amd64', agent_version.architecture)
-
-    def test_get_agent_version_devel(self):
-        control_str = control_string_template.format(
-            '1:2.1~beta1-0ubuntu1~16.04.1~juju1')
-        agent_version = get_agent_version(control_str)
-        self.assertEqual('2.1-beta1', agent_version.version)
-        self.assertEqual('2.1', agent_version.major_minor)
-        self.assertEqual('xenial', agent_version.series)
-        self.assertEqual('amd64', agent_version.architecture)
+class TestMakeUbuntuAgent(TestCase):
+
+    def test_make_ubuntu_agent(self):
+        with temp_dir() as workspace:
+            dest_debs = os.path.join(workspace, 'debs')
+            agent_dir = os.path.join(dest_debs, 'agent', '1.25.5')
+            os.makedirs(agent_dir)
+            stanzas = os.path.join(workspace, 'stanzas')
+            os.mkdir(stanzas)
+            agent = os.path.join(dest_debs, 'juju-1.25.5-ubuntu-amd64.tgz')
+            with open(agent, 'w') as dummy_file:
+                dummy_file.write('ubuntu agent')
+            make_ubuntu_agent(dest_debs, 'proposed', '1.25.5')
+            agent_path = os.path.join(
+                workspace, 'debs', 'agent', '1.25.5',
+                'juju-1.25.5-ubuntu-amd64.tgz')
+            self.assertTrue(os.path.exists(agent_path))
+            stanza_path = os.path.join(
+                workspace, 'debs', 'proposed-1.25.5-ubuntu.json')
+            self.assertTrue(os.path.exists(stanza_path))
 
 
 class TestMakeCentosAgent(TestCase):