Merge lp:~abentley/juju-ci-tools/industrial-agent-stream into lp:juju-ci-tools

[Aaron Bentley]

Add --agent-stream to industrial_test.

Add --agent-stream support to industrial_test.

This allows running industrial tests of arbitrary builds without the candidate urls.

[Curtis Hovey (sinzui)]

thank you.

review: Approve (code)

Unmerged revisions

1244. By Aaron Bentley on 2016-01-14

Fix lint.

=== modified file 'industrial_test.py'
--- industrial_test.py	2016-01-13 21:31:55 +0000
+++ industrial_test.py	2016-01-14 21:01:47 +0000
@@ -72,6 +72,7 @@
     :ivar new_juju_path: Path to the non-system juju.
     :ivar stages: A list of StageAttempts.
     :ivar attempt_count: The number of attempts needed for each stage.
+    :ivar agent_stream: The agent stream to use for testing.
     """
 
     @classmethod
@@ -80,7 +81,8 @@
         stages = cls.get_stages(suite, config)
         return cls(args.env, args.new_juju_path,
                    stages, args.log_dir, args.attempts, args.attempts * 2,
-                   args.new_agent_url, args.debug, args.old_stable)
+                   args.new_agent_url, args.debug, args.old_stable,
+                   args.agent_stream)
 
     @staticmethod
     def get_stages(suite, config):
@@ -88,7 +90,7 @@
 
     def __init__(self, env, new_juju_path, stages, log_dir, attempt_count=2,
                  max_attempts=1, new_agent_url=None, debug=False,
-                 really_old_path=None):
+                 really_old_path=None, agent_stream=None):
         self.env = env
         self.really_old_path = really_old_path
         self.new_juju_path = new_juju_path
@@ -98,6 +100,7 @@
         self.max_attempts = max_attempts
         self.debug = debug
         self.log_parent_dir = log_dir
+        self.agent_stream = agent_stream
 
     def make_results(self):
         """Return a results list for use in run_tests."""
@@ -150,7 +153,8 @@
         paths = [self.really_old_path, stable_path, self.new_juju_path]
         upgrade_sequence = [p for p in paths if p is not None]
         stage_attempts = [self.stages.factory(upgrade_sequence,
-                                              self.log_parent_dir)]
+                                              self.log_parent_dir,
+                                              self.agent_stream)]
         return IndustrialTest.from_args(self.env, self.new_juju_path,
                                         stage_attempts, self.new_agent_url,
                                         self.debug)
@@ -273,7 +277,7 @@
     """
 
     @classmethod
-    def factory(cls, upgrade_sequence):
+    def factory(cls, upgrade_sequence, agent_stream):
         return cls()
 
     @staticmethod
@@ -428,7 +432,7 @@
         return dict([cls.prepare_upgrade.as_tuple()])
 
     @classmethod
-    def factory(cls, upgrade_sequence):
+    def factory(cls, upgrade_sequence, agent_stream):
         if len(upgrade_sequence) < 2:
             raise ValueError('Not enough paths for upgrade.')
         bootstrap_paths = dict(
@@ -821,14 +825,15 @@
         result.update(DestroyEnvironmentAttempt.get_test_info())
         return result
 
-    def factory(self, upgrade_sequence, log_dir):
+    def factory(self, upgrade_sequence, log_dir, agent_stream):
         """Emit an AttemptSuite.
 
         :param upgrade_sequence: The sequence of jujus to upgrade, for
             UpgradeJujuAttempt.
         :param log_dir: Directory to store logs and other artifacts in.
+        :param agent_stream: The agent stream to use for testing.
         """
-        return AttemptSuite(self, upgrade_sequence, log_dir)
+        return AttemptSuite(self, upgrade_sequence, log_dir, agent_stream)
 
 
 class AttemptSuite(SteppedStageAttempt):
@@ -839,12 +844,14 @@
     :ivar upgrade_sequence: The sequence of jujus to upgrade, for
         UpgradeJujuAttempt.
     :ivar log_dir: Directory to store logs and other artifacts in.
+    :ivar agent_stream: The agent stream to use for testing.
     """
 
-    def __init__(self, attempt_list, upgrade_sequence, log_dir):
+    def __init__(self, attempt_list, upgrade_sequence, log_dir, agent_stream):
         self.attempt_list = attempt_list
         self.upgrade_sequence = upgrade_sequence
         self.log_dir = log_dir
+        self.agent_stream = agent_stream
 
     def get_test_info(self):
         """Describe the tests provided by this Stage."""
@@ -862,15 +869,16 @@
         The actual generator is _iter_bs_manager_steps, to simplify testing.
         """
         bootstrap_attempt = self.attempt_list.bootstrap_attempt.factory(
-            self.upgrade_sequence)
+            self.upgrade_sequence, self.agent_stream)
         bs_client = bootstrap_attempt.get_bootstrap_client(client)
         bs_jes_enabled = bs_client.is_jes_enabled()
         jes_enabled = client.is_jes_enabled()
         bs_manager = BootstrapManager(
             client.env.environment, bs_client, bs_client,
             bootstrap_host=None,
-            machines=[], series=None, agent_url=None, agent_stream=None,
-            region=None, log_dir=make_log_dir(self.log_dir), keep_env=True,
+            machines=[], series=None, agent_url=None,
+            agent_stream=self.agent_stream, region=None,
+            log_dir=make_log_dir(self.log_dir), keep_env=True,
             permanent=jes_enabled, jes_enabled=bs_jes_enabled)
         return self._iter_bs_manager_steps(bs_manager, client,
                                            bootstrap_attempt, jes_enabled)
@@ -890,8 +898,9 @@
                 bs_manager.client = client
                 bs_manager.tear_down_client = client
                 bs_manager.jes_enabled = jes_enabled
-                attempts = [a.factory(self.upgrade_sequence) for a in
-                            self.attempt_list.attempt_list]
+                attempts = [
+                    a.factory(self.upgrade_sequence, self.agent_stream)
+                    for a in self.attempt_list.attempt_list]
                 yield self.attempt_list.prepare_suite.as_result(True)
                 for attempt in attempts:
                     for result in attempt.iter_steps(client):
@@ -958,6 +967,8 @@
     parser.add_argument('--new-agent-url')
     parser.add_argument('--single', action='store_true')
     parser.add_argument('--debug', action='store_true', default=False)
+    parser.add_argument('--agent-stream',
+                        help='Agent stream to use for tests.')
     parser.add_argument(
         '--old-stable', help='Path to a version of juju that stable can'
         ' upgrade from.')
@@ -981,7 +992,8 @@
     for suite in args.suite:
         factory = MultiIndustrialTest.get_stages(suite, env.config)
         upgrade_sequence = [upgrade_client.full_path, client.full_path]
-        suite = factory.factory(upgrade_sequence, args.log_dir)
+        suite = factory.factory(upgrade_sequence, args.log_dir,
+                                args.agent_stream)
         steps_iter = suite.iter_steps(client)
         for step in steps_iter:
             print(step)

=== modified file 'run-reliability-test.bash'
--- run-reliability-test.bash	2015-03-02 21:22:28 +0000
+++ run-reliability-test.bash	2016-01-14 21:01:47 +0000
@@ -15,11 +15,13 @@
 fi
 set -x
 # Delete all files in $WORKSPACE, but no error if empty.
-find $WORKSPACE -type f -delete
+find $WORKSPACE -mindepth 1 -delete
+logs=$WORKSPACE/logs
+mkdir $logs
 $SCRIPTS/write_industrial_test_metadata.py $new_juju_dir/buildvars.json \
   $environment metadata.json
 s3cmd -c $s3cfg put metadata.json $s3base-metadata.json
 timeout -sINT -k 10m 1d $SCRIPTS/industrial_test.py $environment $new_juju \
-  --old-stable $old_stable_juju $suite --attempts $attempts \
+  $logs --old-stable $old_stable_juju $suite --attempts $attempts \
   --json-file results.json $extra_args
 s3cmd -c $s3cfg put results.json $s3base-results.json

=== modified file 'tests/test_industrial_test.py'
--- tests/test_industrial_test.py	2016-01-13 21:33:11 +0000
+++ tests/test_industrial_test.py	2016-01-14 21:01:47 +0000
@@ -164,6 +164,7 @@
         self.assertEqual(args.new_juju_path, 'new-juju')
         self.assertEqual(args.log_dir, 'log-dir')
         self.assertEqual(args.suite, [QUICK])
+        self.assertIs(args.agent_stream, None)
 
     def test_parse_args_attempts(self):
         args = parse_args(['rai', 'new-juju', QUICK, 'log-dir'])
@@ -221,6 +222,13 @@
         args = parse_args(['rai', 'new-juju', QUICK, 'log-dir'])
         self.assertIs(args.old_stable, None)
 
+    def test_parse_args_agent_stream(self):
+        args = parse_args(['rai', 'new-juju', QUICK, 'log-dir',
+                           '--agent-stream', 'asdf'])
+        self.assertEqual(args.agent_stream, 'asdf')
+        args = parse_args(['rai', 'new-juju', QUICK, 'log-dir'])
+        self.assertIs(args.old_stable, None)
+
 
 class FakeStepAttempt:
 
@@ -265,7 +273,7 @@
     normal methods on FakeAttemptClass.
     """
 
-    def factory(self, upgrade_sequence):
+    def factory(self, upgrade_sequence, attempt_stream):
         return self()
 
     def __init__(self, title, *result, **kwargs):
@@ -301,7 +309,7 @@
         args = Namespace(
             env='foo', new_juju_path='new-path', attempts=7, suite=[DENSITY],
             log_dir='log-dir', new_agent_url=None, debug=False,
-            old_stable=None)
+            old_stable=None, agent_stream=None)
         with temp_env('foo'):
             mit = MultiIndustrialTest.from_args(args, QUICK)
         self.assertEqual(mit.env, 'foo')
@@ -309,12 +317,13 @@
         self.assertEqual(mit.attempt_count, 7)
         self.assertEqual(mit.max_attempts, 14)
         self.assertEqual(mit.log_parent_dir, 'log-dir')
+        self.assertIs(mit.agent_stream, None)
         self.assertEqual(
             mit.stages, AttemptSuiteFactory([]))
         args = Namespace(
             env='bar', new_juju_path='new-path2', attempts=6, suite=[FULL],
             log_dir='log-dir2', new_agent_url=None, debug=False,
-            old_stable=None)
+            old_stable=None, agent_stream=None)
         with temp_env('bar'):
             mit = MultiIndustrialTest.from_args(args, FULL)
         self.assertEqual(mit.env, 'bar')
@@ -322,6 +331,7 @@
         self.assertEqual(mit.attempt_count, 6)
         self.assertEqual(mit.max_attempts, 12)
         self.assertEqual(mit.log_parent_dir, 'log-dir2')
+        self.assertIs(mit.agent_stream, None)
         self.assertEqual(
             mit.stages, AttemptSuiteFactory([
                 UpgradeCharmAttempt, DeployManyAttempt,
@@ -330,7 +340,8 @@
     def test_from_args_maas(self):
         args = Namespace(
             env='foo', new_juju_path='new-path', log_dir='log-dir',
-            attempts=7, new_agent_url=None, debug=False, old_stable=None)
+            attempts=7, new_agent_url=None, debug=False, old_stable=None,
+            agent_stream=None)
         with temp_env('foo', {'type': 'maas'}):
             mit = MultiIndustrialTest.from_args(args, DENSITY)
         self.assertEqual(
@@ -339,7 +350,8 @@
     def test_from_args_debug(self):
         args = Namespace(
             env='foo', new_juju_path='new-path', log_dir='log-dir',
-            attempts=7, new_agent_url=None, debug=False, old_stable=None)
+            attempts=7, new_agent_url=None, debug=False, old_stable=None,
+            agent_stream=None)
         with temp_env('foo', {'type': 'maas'}):
             mit = MultiIndustrialTest.from_args(args, DENSITY)
             self.assertEqual(mit.debug, False)
@@ -351,17 +363,30 @@
         args = Namespace(
             env='foo', new_juju_path='new-path', log_dir='log-dir',
             attempts=7, new_agent_url=None, debug=False,
-            old_stable='really-old-path')
+            old_stable='really-old-path', agent_stream=None)
         with temp_env('foo'):
             mit = MultiIndustrialTest.from_args(args, FULL)
         self.assertEqual(mit.really_old_path, 'really-old-path')
         args = Namespace(
             env='bar', new_juju_path='new-path2', log_dir='log-dir',
-            attempts=6, new_agent_url=None, debug=False, old_stable=None)
+            attempts=6, new_agent_url=None, debug=False, old_stable=None,
+            agent_stream=None)
         with temp_env('bar'):
             mit = MultiIndustrialTest.from_args(args, FULL)
         self.assertIs(mit.really_old_path, None)
 
+    def test_from_args_agent_stream(self):
+        args = Namespace(
+            env='foo', new_juju_path='new-path', log_dir='log-dir',
+            attempts=7, new_agent_url=None, debug=False, old_stable=None,
+            agent_stream='foo-stream')
+        with temp_env('foo', {'type': 'maas'}):
+            mit = MultiIndustrialTest.from_args(args, DENSITY)
+            self.assertEqual(mit.debug, False)
+            args.debug = True
+            mit = MultiIndustrialTest.from_args(args, DENSITY)
+            self.assertEqual(mit.agent_stream, 'foo-stream')
+
     def test_get_stages(self):
         self.assertEqual(
             MultiIndustrialTest.get_stages(QUICK, {'type': 'foo'}),
@@ -399,7 +424,7 @@
         args = Namespace(
             env='foo', new_juju_path='new-path', attempts=7,
             log_dir='log-dir', new_agent_url=None, debug=False,
-            old_stable=None)
+            old_stable=None, agent_stream=None)
         with temp_env('foo'):
             mit = MultiIndustrialTest.from_args(args, DENSITY)
         self.assertEqual(
@@ -409,7 +434,7 @@
         args = Namespace(
             env='foo', new_juju_path='new-path', attempts=7,
             log_dir='log-dir', new_agent_url=None, debug=False,
-            old_stable=None)
+            old_stable=None, agent_stream=None)
         with temp_env('foo'):
             mit = MultiIndustrialTest.from_args(args, BACKUP)
         self.assertEqual(
@@ -419,7 +444,7 @@
         args = Namespace(
             env='foo', new_juju_path='new-path', attempts=7,
             log_dir='log-dir', new_agent_url='http://example.net',
-            debug=False, old_stable=None)
+            debug=False, old_stable=None, agent_stream=None)
         with temp_env('foo'):
             mit = MultiIndustrialTest.from_args(args, suite=QUICK)
         self.assertEqual(mit.new_agent_url, 'http://example.net')
@@ -872,7 +897,7 @@
             FakeAttemptClass('foo', False, True, new_path='bar-path'),
             FakeAttemptClass('bar', True, True, new_path='bar-path')])
         log_dir = use_context(self, temp_dir())
-        suite = suite_factory.factory([], log_dir)
+        suite = suite_factory.factory([], log_dir, None)
         industrial = IndustrialTest(old_client, new_client, [suite])
         with patch('industrial_test.BootstrapManager',
                    fake_bootstrap_manager):
@@ -891,7 +916,7 @@
         suite_factory = AttemptSuiteFactory([
             FakeAttemptClass('foo', True, False, new_path='bar-path'),
             FakeAttemptClass('bar', True, True, new_path='bar-path')])
-        suite = suite_factory.factory([], log_dir)
+        suite = suite_factory.factory([], log_dir, None)
         industrial = IndustrialTest(old_client, new_client, [suite])
         with patch('industrial_test.BootstrapManager',
                    fake_bootstrap_manager):
@@ -909,7 +934,8 @@
         log_dir = use_context(self, temp_dir())
         suite = AttemptSuiteFactory([
             FakeAttemptClass('foo', False, False),
-            FakeAttemptClass('bar', True, True)]).factory([], log_dir)
+            FakeAttemptClass('bar', True, True)]).factory([], log_dir,
+                                                          'foo-stream')
         industrial = IndustrialTest(old_client, new_client, [suite])
         with patch('industrial_test.BootstrapManager',
                    fake_bootstrap_manager):
@@ -968,7 +994,7 @@
         new_client = FakeJujuClient()
         attempt = FakeStepAttempt.from_result(True, True)
         log_dir = use_context(self, temp_dir())
-        suite = AttemptSuiteFactory([attempt]).factory([], log_dir)
+        suite = AttemptSuiteFactory([attempt]).factory([], log_dir, None)
         industrial = IndustrialTest(old_client, new_client,
                                     [suite])
 
@@ -1145,7 +1171,7 @@
             def __init__(self):
                 super(StubSA, self).__init__()
 
-        self.assertIs(type(StubSA.factory(['a', 'b', 'c'])), StubSA)
+        self.assertIs(type(StubSA.factory(['a', 'b', 'c'], None)), StubSA)
 
     def test_get_test_info(self):
 
@@ -1670,21 +1696,22 @@
 class TestPrepareUpgradeJujuAttempt(JujuPyTestCase):
 
     def test_factory(self):
-        uj_attempt = PrepareUpgradeJujuAttempt.factory(['a', 'b', 'c'])
+        uj_attempt = PrepareUpgradeJujuAttempt.factory(
+            ['a', 'b', 'c'], None)
         self.assertIs(type(uj_attempt), PrepareUpgradeJujuAttempt)
         self.assertEqual(uj_attempt.bootstrap_paths, {'b': 'a', 'c': 'b'})
 
     def test_factory_empty(self):
         with self.assertRaisesRegexp(
                 ValueError, 'Not enough paths for upgrade.'):
-            PrepareUpgradeJujuAttempt.factory(['a'])
+            PrepareUpgradeJujuAttempt.factory(['a'], None)
         with self.assertRaisesRegexp(
                 ValueError, 'Not enough paths for upgrade.'):
-            PrepareUpgradeJujuAttempt.factory([])
+            PrepareUpgradeJujuAttempt.factory([], None)
 
     def test_get_bootstrap_client(self):
         client = FakeJujuClient(full_path='c', debug=True)
-        puj_attempt = PrepareUpgradeJujuAttempt.factory(['a', 'b', 'c'])
+        puj_attempt = PrepareUpgradeJujuAttempt.factory(['a', 'b', 'c'], None)
         bootstrap_client = puj_attempt.get_bootstrap_client(client)
         self.assertIsNot(bootstrap_client, client)
         self.assertIs(client.debug, bootstrap_client.debug)
@@ -1918,10 +1945,11 @@
         fake_bootstrap = FakeAttemptClass('bootstrap')
         factory = AttemptSuiteFactory([],
                                       bootstrap_attempt=fake_bootstrap)
-        attempt_suite = factory.factory(['1', '2'], 'log-1')
+        attempt_suite = factory.factory(['1', '2'], 'log-1', 'foo-stream')
         self.assertEqual(factory, attempt_suite.attempt_list)
         self.assertEqual(['1', '2'], attempt_suite.upgrade_sequence)
         self.assertEqual('log-1', attempt_suite.log_dir)
+        self.assertEqual('foo-stream', attempt_suite.agent_stream)
 
     def test_get_test_info(self):
         fake_bootstrap = FakeAttemptClass('fake-bootstrap')
@@ -1950,7 +1978,7 @@
         fake_2 = FakeAttemptClass('fake-2')
         factory = AttemptSuiteFactory([fake_1, fake_2],
                                       bootstrap_attempt=fake_bootstrap)
-        attempt_suite = AttemptSuite(factory, None, None)
+        attempt_suite = AttemptSuite(factory, None, None, None)
         self.assertEqual(OrderedDict([
             ('fake-bootstrap-id', {'title': 'fake-bootstrap'}),
             ('prepare-suite', {'title': 'Prepare suite tests',
@@ -1963,24 +1991,41 @@
                                  'report_on': True}),
             ]), attempt_suite.get_test_info())
 
+    @contextmanager
+    def iter_steps_cxt(self, attempt_suite):
+        with patch('industrial_test.BootstrapManager') as mock_bm:
+            with patch.object(attempt_suite,
+                              '_iter_bs_manager_steps') as mock_ibms:
+                with patch('industrial_test.make_log_dir',
+                           return_value='qux-1'):
+                    yield (mock_ibms, mock_bm)
+
     def test_iter_steps(self):
         fake_bootstrap = FakeAttemptClass('fake-bootstrap', '1', '2')
-        with temp_dir() as log_dir:
-            factory = AttemptSuiteFactory([], bootstrap_attempt=fake_bootstrap)
-            attempt_suite = AttemptSuite(factory, None, log_dir)
-            client = FakeJujuClient()
-            with patch('industrial_test.BootstrapManager') as mock_bm:
-                with patch.object(attempt_suite,
-                                  '_iter_bs_manager_steps') as mock_ibms:
-                    iterator = attempt_suite.iter_steps(client)
+        factory = AttemptSuiteFactory([], bootstrap_attempt=fake_bootstrap)
+        attempt_suite = AttemptSuite(factory, None, 'asdf', None)
+        with self.iter_steps_cxt(attempt_suite) as (mock_ibms, mock_bm):
+            client = FakeJujuClient()
+            attempt_suite.iter_steps(client)
+        mock_bm.assert_called_once_with(
+            'name', client, client, agent_stream=None, agent_url=None,
+            bootstrap_host=None, jes_enabled=False, keep_env=True,
+            log_dir='qux-1', machines=[], permanent=False,
+            region=None, series=None)
+
+    def test_iter_steps_agent_stream(self):
+        fake_bootstrap = FakeAttemptClass('fake-bootstrap', '1', '2')
+        factory = AttemptSuiteFactory([], bootstrap_attempt=fake_bootstrap)
+        attempt_suite = AttemptSuite(factory, None, 'asdf', 'bar-stream')
+        with self.iter_steps_cxt(attempt_suite) as (mock_ibms, mock_bm):
+            client = FakeJujuClient()
+            iterator = attempt_suite.iter_steps(client)
         self.assertEqual(iterator, mock_ibms.return_value)
         mock_bm.assert_called_once_with(
-            'name', client, client, agent_stream=None, agent_url=None,
+            'name', client, client, agent_stream='bar-stream', agent_url=None,
             bootstrap_host=None, jes_enabled=False, keep_env=True,
-            log_dir=os.path.join(log_dir, '0'), machines=[], permanent=False,
+            log_dir='qux-1', machines=[], permanent=False,
             region=None, series=None)
-        mock_ibms.assert_called_once_with(mock_bm.return_value, client,
-                                          fake_bootstrap(), False)
 
     def test__iter_bs_manager_steps(self):
         fake_bootstrap = FakeAttemptClass('fake-bootstrap', '1', '2')
@@ -1988,7 +2033,7 @@
         fake_2 = FakeAttemptClass('fake-2', '1', '2')
         factory = AttemptSuiteFactory([fake_1, fake_2],
                                       bootstrap_attempt=fake_bootstrap)
-        attempt_suite = AttemptSuite(factory, None, None)
+        attempt_suite = AttemptSuite(factory, None, None, None)
         client = FakeJujuClient()
         bs_manager = FakeBootstrapManager(client)
         steps = list(attempt_suite._iter_bs_manager_steps(