Merge lp:~nskaggs/juju-ci-tools/add-assess-budget into lp:juju-ci-tools

[Nicholas Skaggs]

This change adds test for the budget commands within juju. Note the following bugs which affect the ability of this test to run. bug 1665013 and bug 1663258

You will need to authenticate to jujucharms.com using charm login, or setting the macaroon yourself with juju.

[Christopher Lee (veebers)]

Please see inline comments.

review: Needs Fixing

[lp:~nskaggs/juju-ci-tools/add-assess-budget]

1899. By Nicholas Skaggs on 2017-02-16

addressed review comments

[Nicholas Skaggs (nskaggs)]

See inline comments. I'm going to have a look at the whole number vs float issue.

[Christopher Lee (veebers)]

Replied to the incline comment replies.

[Christopher Lee (veebers)]

LGTM, a couple of comments re: test naming but nothing blocker.

review: Approve

[lp:~nskaggs/juju-ci-tools/add-assess-budget]

1900. By Nicholas Skaggs on 2017-02-16

final review tweaks

=== added file 'assess_budget.py'
--- assess_budget.py	1970-01-01 00:00:00 +0000
+++ assess_budget.py	2017-02-16 21:51:25 +0000
@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+"""
+This tests the budget commands utilized for commercial charm billing.
+These commands are linked to a ubuntu sso account, and as such, require the
+user account to be setup before test execution (including authentication).
+You can use charm login to do this, or let juju authenticate with a browser.
+"""
+
+from __future__ import print_function
+
+import argparse
+import json
+import logging
+from random import randint
+import subprocess
+import sys
+
+
+from jujupy import (
+    client_from_config,
+)
+from utility import (
+    add_basic_testing_arguments,
+    JujuAssertionError,
+    configure_logging,
+)
+
+__metaclass__ = type
+
+
+log = logging.getLogger("assess_budget")
+
+
+def _get_new_budget_limit(client):
+    """Return availible limit for new budget"""
+    budgets = json.loads(list_budgets(client))
+    limit = int(budgets['total']['limit'])
+    credit = int(budgets['credit'])
+    log.debug('Found credit limit {}, currently used {}'.format(
+        credit, limit))
+    return credit - limit
+
+
+def _get_budgets(client):
+    return json.loads(list_budgets(client))['budgets']
+
+
+def _set_budget_value_expectations(expected_budgets, name, value):
+    # Update our expectations accordingly
+    for budget in expected_budgets:
+        if budget['budget'] == name:
+            # For now, we assume we aren't spending down the budget
+            budget['limit'] = value
+            budget['unallocated'] = value
+            # .00 is appended to availible for some reason
+            budget['available'] = '{:.2f}'.format(float(value))
+            log.info('Expected budget updated: "{}" to {}'.format(name, value))
+
+
+def _try_setting_budget(client, name, value):
+    try:
+        output = set_budget(client, name, value)
+    except subprocess.CalledProcessError as e:
+        output = [e.output, getattr(e, 'stderr', '')]
+        raise JujuAssertionError('Could not set budget {}'.format(output))
+
+    if 'budget limit updated' not in output:
+        raise JujuAssertionError('Error calling set-budget {}'.format(output))
+
+
+def _try_creating_budget(client, name, value):
+    try:
+        create_budget(client, name, value)
+        log.info('Created new budget "{}" with value {}'.format(name,
+                                                                value))
+    except subprocess.CalledProcessError as e:
+        output = [e.output, getattr(e, 'stderr', '')]
+        if any('already exists' in message for message in output):
+            log.info('Reusing budget "{}" with value {}'.format(name, value))
+            pass  # this will be a failure once lp:1663258 is fixed
+        else:
+            raise JujuAssertionError(
+                'Error testing create-budget: {}'.format(output))
+    except:
+        raise JujuAssertionError('Added duplicate budget')
+
+
+def _try_greater_than_limit_budget(client, name, limit):
+    error_strings = {
+        'pass': 'exceed the credit limit',
+        'unknown': 'Error testing budget greater than credit limit',
+        'fail': 'Credit limit exceeded'
+    }
+    over_limit_value = str(limit + randint(1, 100))
+    assert_set_budget(client, name, over_limit_value, error_strings)
+
+
+def _try_negative_budget(client, name):
+    error_strings = {
+        'pass': 'Could not set budget',
+        'unknown': 'Error testing negative budget',
+        'fail': 'Negative budget allowed'
+    }
+    negative_budget_value = str(randint(-1000, -1))
+    assert_set_budget(client, name, negative_budget_value, error_strings)
+
+
+def assert_sorted_equal(found, expected):
+    found = sorted(found)
+    expected = sorted(expected)
+    if found != expected:
+        raise JujuAssertionError(
+            'Found: {}\nExpected: {}'.format(found, expected))
+
+
+def assert_set_budget(client, name, limit, error_strings):
+    try:
+        _try_setting_budget(client, name, limit)
+    except JujuAssertionError as e:
+        if error_strings['pass'] not in e.message:
+            raise JujuAssertionError(
+                '{}: {}'.format(error_strings['unknown'], e))
+    else:
+        raise JujuAssertionError(error_strings['fail'])
+
+
+def create_budget(client, name, value):
+    """Create a budget"""
+    return client.get_juju_output('create-budget', name, value,
+                                  include_e=False)
+
+
+def list_budgets(client):
+    """Return defined budgets as json."""
+    return client.get_juju_output('list-budgets', '--format', 'json',
+                                  include_e=False)
+
+
+def set_budget(client, name, value):
+    """Change an existing budgets allocation."""
+    return client.get_juju_output('set-budget', name, value, include_e=False)
+
+
+def show_budget(client, name):
+    """Return specified budget as json."""
+    return client.get_juju_output('show-budget', name, '--format', 'json',
+                                  include_e=False)
+
+
+def assess_budget(client):
+    # Since we can't remove budgets until lp:1663258
+    # is fixed, we avoid creating new random budgets and hardcode.
+    # We also, zero out the previous budget
+    budget_name = 'test'
+    _try_setting_budget(client, budget_name, '0')
+
+    budget_limit = _get_new_budget_limit(client)
+    assess_budget_limit(budget_limit)
+
+    expected_budgets = _get_budgets(client)
+    budget_value = str(randint(1, budget_limit / 2))
+    assess_create_budget(client, budget_name, budget_value, budget_limit)
+
+    budget_value = str(randint(budget_limit / 2 + 1, budget_limit))
+    assess_set_budget(client, budget_name, budget_value, budget_limit)
+    assess_show_budget(client, budget_name, budget_value)
+
+    _set_budget_value_expectations(expected_budgets, budget_name, budget_value)
+    assess_list_budgets(client, expected_budgets)
+
+
+def assess_budget_limit(budget_limit):
+    log.info('Assessing budget limit {}'.format(budget_limit))
+
+    if budget_limit < 0:
+        raise JujuAssertionError(
+            'Negative Budget Limit {}'.format(budget_limit))
+
+
+def assess_create_budget(client, budget_name, budget_value, budget_limit):
+    """Test create-budget command"""
+    log.info('create-budget "{}" with value {}, limit {}'.format(budget_name,
+                                                                 budget_value,
+                                                                 budget_limit))
+
+    # Do this twice, to ensure budget exists and we can check for
+    # duplicate message. Ideally, once lp:1663258 is fixed, we will
+    # assert on initial budget creation as well.
+    _try_creating_budget(client, budget_name, budget_value)
+
+    log.info('Trying duplicate create-budget')
+    _try_creating_budget(client, budget_name, budget_value)
+
+
+def assess_list_budgets(client, expected_budgets):
+    log.info('list-budgets testing expected values')
+    # Since we can't remove budgets until lp:1663258
+    # is fixed, we don't modify the list contents or count
+    # Nonetheless, we assert on it for future use
+    budgets = _get_budgets(client)
+    assert_sorted_equal(budgets, expected_budgets)
+
+
+def assess_set_budget(client, budget_name, budget_value, budget_limit):
+    """Test set-budget command"""
+    log.info('set-budget "{}" with value {}, limit {}'.format(budget_name,
+                                                              budget_value,
+                                                              budget_limit))
+    _try_setting_budget(client, budget_name, budget_value)
+
+    # Check some bounds
+    # Since budgetting is important, and the functional test is cheap,
+    # let's test some basic bounds
+    log.info('Trying set-budget with value greater than budget limit')
+    _try_greater_than_limit_budget(client, budget_name, budget_limit)
+
+    log.info('Trying set-budget with negative value')
+    _try_negative_budget(client, budget_name)
+
+
+def assess_show_budget(client, budget_name, budget_value):
+    log.info('show-budget "{}" with value {}'.format(budget_name,
+                                                     budget_value))
+
+    budget = json.loads(show_budget(client, budget_name))
+
+    # assert budget value
+    if budget['limit'] != budget_value:
+        raise JujuAssertionError('Budget limit found {}, expected {}'.format(
+            budget['limit'], budget_value))
+
+    # assert on usage (0% until we use it)
+    if budget['total']['usage'] != '0%':
+        raise JujuAssertionError('Budget usage found {}, expected {}'.format(
+            budget['total']['usage'], '0%'))
+
+
+def parse_args(argv):
+    """Parse all arguments."""
+    parser = argparse.ArgumentParser(description="Test budget commands")
+    add_basic_testing_arguments(parser)
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    configure_logging(args.verbose)
+    client = client_from_config(args.env, args.juju_bin, False)
+    assess_budget(client)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

=== added file 'tests/test_assess_budget.py'
--- tests/test_assess_budget.py	1970-01-01 00:00:00 +0000
+++ tests/test_assess_budget.py	2017-02-16 21:51:25 +0000
@@ -0,0 +1,282 @@
+"""Tests for assess_budget module."""
+
+import logging
+import json
+from random import randint
+import StringIO
+from subprocess import CalledProcessError
+
+from mock import (
+    Mock,
+    patch,
+)
+
+from assess_budget import (
+    _try_greater_than_limit_budget,
+    _try_negative_budget,
+    assess_budget,
+    assess_budget_limit,
+    assess_create_budget,
+    assess_list_budgets,
+    assess_set_budget,
+    assess_show_budget,
+    main,
+    parse_args,
+)
+from jujupy import (
+    fake_juju_client,
+)
+from tests import (
+    parse_error,
+    TestCase,
+)
+from utility import (
+    JujuAssertionError,
+)
+
+
+class TestParseArgs(TestCase):
+
+    def test_common_args(self):
+        args = parse_args(["an-env", "/bin/juju", "/tmp/logs", "an-env-mod"])
+        self.assertEqual("an-env", args.env)
+        self.assertEqual("/bin/juju", args.juju_bin)
+        self.assertEqual("/tmp/logs", args.logs)
+        self.assertEqual("an-env-mod", args.temp_env_name)
+        self.assertEqual(False, args.debug)
+
+    def test_help(self):
+        fake_stdout = StringIO.StringIO()
+        with parse_error(self) as fake_stderr:
+            with patch("sys.stdout", fake_stdout):
+                parse_args(["--help"])
+        self.assertEqual("", fake_stderr.getvalue())
+
+
+class TestMain(TestCase):
+
+    def test_main(self):
+        argv = ["an-env", "/bin/juju", "/tmp/logs", "an-env-mod", "--verbose"]
+        client = Mock(spec=["is_jes_enabled"])
+        with patch('assess_budget.subprocess.check_output',
+                   autospec=True):
+            with patch("assess_budget.configure_logging",
+                       autospec=True) as mock_cl:
+                with patch('assess_budget.client_from_config',
+                           return_value=client) as mock_cfc:
+                    with patch("assess_budget.assess_budget",
+                               autospec=True) as mock_assess:
+                        main(argv)
+        mock_cl.assert_called_once_with(logging.DEBUG)
+        mock_cfc.assert_called_once_with('an-env', "/bin/juju", False)
+        mock_assess.assert_called_once_with(client)
+
+
+class TestAssess(TestCase):
+
+    def setUp(self):
+        super(TestAssess, self).setUp()
+        self.fake_client = fake_juju_client()
+        self.budget_name = 'test'
+        self.budget_limit = randint(1000, 10000)
+        self.budget_value = str(randint(100, 900))
+
+    def test_assess_budget(self):
+        show_b = patch("assess_budget.assess_show_budget",
+                       autospec=True)
+        list_b = patch("assess_budget.assess_list_budgets",
+                       autospec=True)
+        set_b = patch("assess_budget.assess_set_budget",
+                      autospec=True)
+        create_b = patch("assess_budget.assess_create_budget",
+                         autospec=True)
+        b_limit = patch("assess_budget.assess_budget_limit",
+                        autospec=True)
+        init_b = patch("assess_budget._try_setting_budget",
+                       autospec=True)
+        expect_b = patch("assess_budget._set_budget_value_expectations",
+                         autospec=True)
+        update_e = patch("assess_budget._get_budgets", autospec=True)
+
+        with show_b as show_b_mock, list_b as list_b_mock, \
+            set_b as set_b_mock, create_b as create_b_mock, \
+            b_limit as b_limit_mock, init_b as init_b_mock, \
+                expect_b as expect_b_mock, update_e as update_e_mock:
+                with patch("assess_budget.json.loads"):
+                    with patch("assess_budget.randint",
+                               return_value=self.budget_value):
+                        assess_budget(self.fake_client)
+
+                        init_b_mock.assert_called_once_with(self.fake_client,
+                                                            self.budget_name,
+                                                            '0')
+                        expect_b_mock.assert_called_once_with(
+                            update_e_mock(self.fake_client),
+                            self.budget_name, self.budget_value)
+                        b_limit_mock.assert_called_once_with(0)
+                        create_b_mock.assert_called_once_with(
+                            self.fake_client, self.budget_name,
+                            self.budget_value, 0)
+                        set_b_mock.assert_called_once_with(
+                            self.fake_client, self.budget_name,
+                            self.budget_value, 0)
+                        show_b_mock.assert_called_once_with(
+                            self.fake_client, self.budget_name,
+                            self.budget_value)
+                        list_b_mock.assert_called_once_with(
+                            self.fake_client, update_e_mock(self.fake_client))
+
+                        self.assertEqual(init_b_mock.call_count, 1)
+                        self.assertEqual(expect_b_mock.call_count, 1)
+                        self.assertEqual(b_limit_mock.call_count, 1)
+                        self.assertEqual(create_b_mock.call_count, 1)
+                        self.assertEqual(set_b_mock.call_count, 1)
+                        self.assertEqual(show_b_mock.call_count, 1)
+                        self.assertEqual(list_b_mock.call_count, 1)
+
+
+class TestAssessShowBudget(TestAssess):
+
+    def setUp(self):
+        super(TestAssessShowBudget, self).setUp()
+        self.fake_json = json.loads('{"limit":"' + self.budget_value +
+                                    '","total":{"usage":"0%"}}')
+
+    def test_assess_show_budget(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads",
+                       return_value=self.fake_json):
+                    assess_show_budget(self.fake_client, self.budget_name,
+                                       self.budget_value)
+
+    def test_raises_budget_usage_error(self):
+        error_usage = randint(1, 100)
+        self.fake_json['total']['usage'] = error_usage
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads",
+                       return_value=self.fake_json):
+                with self.assertRaises(JujuAssertionError) as ex:
+                    assess_show_budget(self.fake_client, self.budget_name,
+                                       self.budget_value)
+                self.assertEqual(ex.exception.message,
+                                 'Budget usage found {}, expected 0%'.format(
+                                    error_usage))
+
+    def test_raises_budget_limit_error(self):
+        self.fake_json['limit'] = 0
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads",
+                       return_value=self.fake_json):
+                with self.assertRaises(JujuAssertionError) as ex:
+                    assess_show_budget(self.fake_client, self.budget_name,
+                                       self.budget_value)
+                self.assertEqual(ex.exception.message,
+                                 'Budget limit found 0, expected {}'.format(
+                                     self.budget_value))
+
+
+class TestAssessListBudgets(TestAssess):
+
+    def setUp(self):
+        super(TestAssessListBudgets, self).setUp()
+        snippet = '[{"budget": "test", "limit": "300"}]'
+        self.fake_budgets_json = json.loads('{"budgets":' + snippet + '}')
+        self.fake_budget_json = json.loads(snippet)
+        unexpected_snippet = '[{"budget": "test", "limit": "100"}]'
+        self.fake_unexpected_budgets_json = json.loads(
+            '{"budgets":' + unexpected_snippet + '}')
+        self.fake_unexpected_budget_json = json.loads(unexpected_snippet)
+
+    def test_assess_list_budgets(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads",
+                       return_value=self.fake_budgets_json):
+                assess_list_budgets(self.fake_client, self.fake_budget_json)
+
+    def test_raises_list_mismatch(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads",
+                       return_value=self.fake_unexpected_budgets_json):
+                with self.assertRaises(JujuAssertionError) as ex:
+                    assess_list_budgets(
+                        self.fake_client, self.fake_budget_json)
+                self.assertEqual(ex.exception.message,
+                                 'Found: {}\nExpected: {}'.format(
+                                     self.fake_unexpected_budget_json,
+                                     self.fake_budget_json))
+
+
+class TestAssessSetBudget(TestAssess):
+
+    def test_assess_set_budget(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads"):
+                with patch("assess_budget._try_setting_budget"):
+                    with patch("assess_budget.assert_set_budget"):
+                        assess_set_budget(self.fake_client, self.budget_name,
+                                          self.budget_value, self.budget_limit)
+
+    def test_raises_on_exceed_credit_limit(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads"):
+                with patch("assess_budget._try_setting_budget"):
+                    with self.assertRaises(JujuAssertionError) as ex:
+                        _try_greater_than_limit_budget(self.fake_client,
+                                                       self.budget_name,
+                                                       self.budget_limit)
+                    self.assertEqual(ex.exception.message,
+                                     'Credit limit exceeded')
+
+    def test_raises_on_negative_budget(self):
+        self.budget_limit = -abs(self.budget_limit)
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.json.loads"):
+                with patch("assess_budget._try_setting_budget"):
+                    with self.assertRaises(JujuAssertionError) as ex:
+                        _try_negative_budget(self.fake_client,
+                                             self.budget_name)
+                    self.assertEqual(ex.exception.message,
+                                     'Negative budget allowed')
+
+
+class TestAssessCreateBudget(TestAssess):
+
+    def test_assess_create_budget(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.create_budget"):
+                assess_create_budget(self.fake_client, self.budget_name,
+                                     self.budget_value, self.budget_limit)
+
+    def test_raises_duplicate_budget(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.create_budget",
+                       side_effect=JujuAssertionError):
+                with self.assertRaises(JujuAssertionError) as ex:
+                    assess_create_budget(self.fake_client, self.budget_name,
+                                         self.budget_value, self.budget_limit)
+                self.assertEqual(ex.exception.message,
+                                 'Added duplicate budget')
+
+    def test_raises_creation_error(self):
+        with patch.object(self.fake_client, 'get_juju_output'):
+            with patch("assess_budget.create_budget",
+                       side_effect=CalledProcessError(1, 'foo', 'bar')):
+                with self.assertRaises(JujuAssertionError) as ex:
+                    assess_create_budget(self.fake_client, self.budget_name,
+                                         self.budget_value, self.budget_limit)
+                self.assertEqual(ex.exception.message,
+                                 "Error testing create-budget: ['bar', '']")
+
+
+class TestAssessBudgetLimit(TestAssess):
+
+    def test_assess_budget_limt(self):
+        budget_limit = randint(1, 10000)
+        assess_budget_limit(budget_limit)
+
+    def test_raises_error_on_negative_limit(self):
+        neg_budget_limit = randint(-1000, -1)
+        with self.assertRaises(JujuAssertionError) as ex:
+            assess_budget_limit(neg_budget_limit)
+        self.assertEqual(ex.exception.message,
+                         'Negative Budget Limit {}'.format(neg_budget_limit))