Merge lp:~1chb1n/charms/trusty/cinder/next.1601-test-update1 into lp:~openstack-charmers-archive/charms/trusty/cinder/next

[Ryan Beisner]

Cosmetic updates to tests/dir and Makefile to standardize *os-charms, prep for 16.01. No changes to test code, hooks or charmhelpers.

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #16877 cinder-next for 1chb1n mp282068
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/16877/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #15767 cinder-next for 1chb1n mp282068
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/15767/

[lp:~1chb1n/charms/trusty/cinder/next.1601-test-update1]

141. By Ryan Beisner on 2016-01-08

remove amulet tests for unsupported releases

142. By Ryan Beisner on 2016-01-08

Move 00-setup to prevent extra, unnecessary bootstrap in test runs.

143. By Ryan Beisner on 2016-01-08

Re-number amulet test file names; add missing combos as not-executable for now.

144. By Ryan Beisner on 2016-01-08

Update tests/README

145. By Ryan Beisner on 2016-01-08

Update bundletester testplan yaml file

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #16889 cinder-next for 1chb1n mp282068
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/16889/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #15780 cinder-next for 1chb1n mp282068
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/15780/

[Liam Young (gnuoy)]

Approved

review: Approve

=== modified file 'Makefile'
--- Makefile	2015-10-29 04:46:01 +0000
+++ Makefile	2016-01-08 21:46:50 +0000
@@ -13,6 +13,7 @@
 
 functional_test:
 	@echo Starting amulet deployment tests...
+	@tests/setup/00-setup
 	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
 
 bin/charm_helpers_sync.py:
 
=== added file 'tests/018-basic-trusty-liberty'
--- tests/018-basic-trusty-liberty	1970-01-01 00:00:00 +0000
+++ tests/018-basic-trusty-liberty	2016-01-08 21:46:50 +0000
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic cinder deployment on trusty-liberty."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-liberty',
+                                       source='cloud:trusty-updates/liberty')
+    deployment.run_tests()
 
=== added file 'tests/019-basic-trusty-mitaka'
--- tests/019-basic-trusty-mitaka	1970-01-01 00:00:00 +0000
+++ tests/019-basic-trusty-mitaka	2016-01-08 21:46:50 +0000
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic cinder deployment on trusty-liberty."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-liberty',
+                                       source='cloud:trusty-updates/liberty')
+    deployment.run_tests()
 
=== removed file 'tests/019-basic-vivid-kilo'
--- tests/019-basic-vivid-kilo	2015-06-23 23:52:07 +0000
+++ tests/019-basic-vivid-kilo	1970-01-01 00:00:00 +0000
@@ -1,9 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic Cinder deployment on vivid-kilo."""
-
-from basic_deployment import CinderBasicDeployment
-
-if __name__ == '__main__':
-    deployment = CinderBasicDeployment(series='vivid')
-    deployment.run_tests()
 
=== added file 'tests/020-basic-wily-liberty'
--- tests/020-basic-wily-liberty	1970-01-01 00:00:00 +0000
+++ tests/020-basic-wily-liberty	2016-01-08 21:46:50 +0000
@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder deployment on wily-liberty."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='wily')
+    deployment.run_tests()
 
=== added file 'tests/021-basic-xenial-mitaka'
--- tests/021-basic-xenial-mitaka	1970-01-01 00:00:00 +0000
+++ tests/021-basic-xenial-mitaka	2016-01-08 21:46:50 +0000
@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Cinder deployment on xenial-mitaka."""
+
+from basic_deployment import CinderBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CinderBasicDeployment(series='xenial')
+    deployment.run_tests()
 
=== removed file 'tests/054-basic-vivid-kilo-git'
--- tests/054-basic-vivid-kilo-git	2015-06-23 23:52:07 +0000
+++ tests/054-basic-vivid-kilo-git	1970-01-01 00:00:00 +0000
@@ -1,9 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic Cinder git deployment on vivid-kilo."""
-
-from basic_deployment import CinderBasicDeployment
-
-if __name__ == '__main__':
-    deployment = CinderBasicDeployment(series='vivid', git=True)
-    deployment.run_tests()
 
=== modified file 'tests/README'
--- tests/README	2015-06-23 23:52:07 +0000
+++ tests/README	2016-01-08 21:46:50 +0000
@@ -1,89 +1,113 @@
-This directory provides Amulet tests that focus on verification of Cinder
-deployments.
-
-test_* methods are called in lexical sort order.
-
-Test name convention to ensure desired test order:
+This directory provides Amulet tests to verify basic deployment functionality
+from the perspective of this charm, its requirements and its features, as
+exercised in a subset of the full OpenStack deployment test bundle topology.
+
+Reference:  lp:openstack-charm-testing for full test bundles.
+
+A single topology and configuration is defined and deployed, once for each of
+the defined Ubuntu:OpenStack release combos.  The ongoing goal is for this
+charm to always possess tests and combo definitions for all currently-supported
+release combinations of U:OS.
+
+test_* methods are called in lexical sort order, as with most runners.  However,
+each individual test method should be idempotent and expected to pass regardless
+of run order or Ubuntu:OpenStack combo.  When writing or modifying tests,
+ensure that every individual test is not dependent on another test_ method.
+
+Test naming convention, purely for code organization purposes:
     1xx service and endpoint checks
     2xx relation checks
     3xx config checks
     4xx functional checks
-    9xx restarts and other final checks
+    9xx restarts, config changes, actions and other final checks
 
-In order to run tests, you'll need charm-tools installed (in addition to
-juju, of course):
+In order to run tests, charm-tools and juju must be installed:
     sudo add-apt-repository ppa:juju/stable
     sudo apt-get update
-    sudo apt-get install charm-tools
-
-If you use a web proxy server to access the web, you'll need to set the
-AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.
+    sudo apt-get install charm-tools juju juju-deployer amulet
+
+Alternatively, tests may be exercised with proposed or development versions
+of juju and related tools:
+
+    # juju proposed version
+    sudo add-apt-repository ppa:juju/proposed
+    sudo apt-get update
+    sudo apt-get install charm-tools juju juju-deployer
+
+    # juju development version
+    sudo add-apt-repository ppa:juju/devel
+    sudo apt-get update
+    sudo apt-get install charm-tools juju juju-deployer
+
+Some tests may need to download files. If a web proxy server is required in
+the environment, the AMULET_HTTP_PROXY environment variable must be set and
+passed into the juju test command.  This is unrelated to juju's http proxy
+settings or behavior.
 
 The following examples demonstrate different ways that tests can be executed.
 All examples are run from the charm's root directory.
 
-  * To run all tests (starting with 00-setup):
-
-      make test
-
-  * To run a specific test module (or modules):
-
-      juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
-
-  * To run a specific test module (or modules), and keep the environment
-    deployed after a failure:
-
-      juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
+  * To run all +x tests in the tests directory:
+
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      make functional_test
+
+  * To run the tests against a specific release combo as defined in tests/:
+
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      juju test -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
+
+  * To run tests and keep the juju environment deployed after a failure:
+
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      juju test --set-e -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
 
   * To re-run a test module against an already deployed environment (one
     that was deployed by a previous call to 'juju test --set-e'):
 
-      ./tests/15-basic-trusty-icehouse
-
-
-For debugging and test development purposes, all code should be idempotent.
-In other words, the code should have the ability to be re-run without changing
-the results beyond the initial run.  This enables editing and re-running of a
-test module against an already deployed environment, as described above.
-
-Additional testing and debug information:
+      ./tests/015-basic-trusty-icehouse
+
+  * Even with --set-e, `juju test` will tear down the deployment when all
+    tests pass. The following work flow may be more effective when
+    iterating on test writing.
+
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      ./tests/setup/00-setup
+      juju bootstrap
+      ./tests/015-basic-trusty-icehouse
+      # make some changes, run tests again
+      ./tests/015-basic-trusty-icehouse
+      # make some changes, run tests again
+      ./tests/015-basic-trusty-icehouse
+
+  * There may be test definitions in the tests/ dir which are not set +x
+    executable.  This is generally true for deprecated releases, or for
+    upcoming releases which are not yet validated and enabled.  To enable
+    and run these tests:
+      bzr branch lp:charms/trusty/foo
+      cd foo
+      ls tests
+      chmod +x tests/017-basic-trusty-kilo
+      ./tests/setup/00-setup
+      juju bootstrap
+      ./tests/017-basic-trusty-kilo
+
+
+Additional notes:
 
   * Use DEBUG to turn on debug logging, use ERROR otherwise.
-
       u = OpenStackAmuletUtils(ERROR)
       u = OpenStackAmuletUtils(DEBUG)
 
   * To interact with the deployed environment:
-
       export OS_USERNAME=admin
       export OS_PASSWORD=openstack
       export OS_TENANT_NAME=admin
       export OS_REGION_NAME=RegionOne
       export OS_AUTH_URL=${OS_AUTH_PROTOCOL:-http}://`juju-deployer -e trusty -f keystone`:5000/v2.0
       keystone user-list
-      cinder list
-
-  * Handy commands for watching these tests run or manually poking around:
-
-      watch "cinder list && cinder snapshot-list && glance image-list"
-      watch "juju get cinder | egrep 'verbose|debug|true|false'"
-      watch "juju ssh cinder/0 'sudo lvs && sudo status cinder-api && sudo status cinder-volume'"
-      juju set cinder verbose=false && juju set cinder debug=false
-
-  * Be aware that method names that start with 'test_' are called in alpha/num
-    order.  Test methods which are order-sensitive can be numbered to ensure
-    that they run in the expected order.  ex:
-
-      test_000_earlytest
-      test_300_sometest
-      test_999_latertest
-
-  * Preserving the deployed environment:
-    Even with juju --set-e, amulet will tear down the juju environment
-    when all tests pass.  This force_fail 'test' can be used in basic_deployment.py
-    to simulate a failed test and keep the environment.
-
-    def test_zzzz_fake_fail(self):
-        '''Force a fake fail to keep juju environment after a successful test run'''
-        # Useful in test writing, when used with:   juju test --set-e
-        amulet.raise_status(amulet.FAIL, msg='using fake fail to keep juju environment')
+      glance image-list
 
=== added directory 'tests/setup'
=== renamed file 'tests/00-setup' => 'tests/setup/00-setup'
=== modified file 'tests/tests.yaml'
--- tests/tests.yaml	2015-10-06 14:11:42 +0000
+++ tests/tests.yaml	2016-01-08 21:46:50 +0000
@@ -1,5 +1,5 @@
 bootstrap: true
-reset: true
+reset: false
 virtualenv: true
 makefile:
     - lint
@@ -9,6 +9,7 @@
 packages:
     - amulet
     - distro-info-data
+    - python-ceilometerclient
     - python-cinderclient
     - python-distro-info
     - python-glanceclient