Merge ~chad.smith/cloud-init:tests/integration-simplified-failure-output into cloud-init:master

[Chad Smith]

tests: provide human-readable integration test summary when --verbose

Integration tests will now provide a brief summary for test failures
listed by platform and distribution. The failure summary will only consist
of failed test name and assert error message.

Drop the verbose dictionary of all integration test output because this
content is unreadable given the large number of integration test results
listed within this dictionary.

verbose test output before:

OK (skipped=2)
2018-06-12 18:39:47,003 - tests.cloud_tests - WARNING - test: platform='lxd', os='artful' failed 1 tests
2018-06-12 18:39:47,003 - tests.cloud_tests - DEBUG - verify results: {'lxd': {'artful': {'modules/apt_pipelining_disable': {'passed': True, 'failures': []}, 'modules/runcmd': {'passed': True, 'failures': []}, 'modules/salt_minion': {'passed': True, 'failures': []}, 'modules/user_groups': {'passed': True, 'failures': []}, 'modules/ssh_auth_key_fingerprints_enable': {'passed': True, 'failures': []}, 'modules/bootcmd': {'passed': True, 'failures': []}, 'modules/keys_to_console': {'passed': True, 'failures': []}, 'main/command_output_simple': {'passed': True, 'failures': []}, 'modules/set_password_expire': {'passed': True, 'failures': []}, 'modules/timezone': {'passed': True, 'failures': []}, 'modules/seed_random_data': {'passed': True, 'failures': []}, 'modules/apt_configure_sources_key': {'passed': True, 'failures': []}, 'modules/write_files': {'passed': True, 'failures': []}, 'modules/lxd_dir': {'passed': True, 'failures': []}, 'modules/ntp_servers': {'passed': True, 'failures': []}, 'modules/apt_configure_proxy': {'passed': True, 'failures': []}, 'modules/snappy': {'passed': True, 'failures': []}, 'modules/set_password': {'passed': True, 'failures': []}, 'modules/ssh_import_id': {'passed': True, 'failures': []}, 'modules/ntp_chrony': {'passed': False, 'failures': [{'traceback': 'Traceback (most recent call last):\n File "/var/lib/jenkins/slaves/torkoal/workspace/cloud-init-integration-lxd-a/cloud-init/tests/cloud_tests/testcases/base.py", line 132, in test_no_stages_errors\n .format(status[stage][\'errors\'], stage))\nAssertionError: 1 != 0 : errors [\'(\\\'ntp\\\', ProcessExecutionError("Unexpected error while running command.\\\\nCommand: [\\\'eatmydata\\\', \\\'apt-get\\\', \\\'--option=Dpkg::Options::=--force-confold\\\', \\\'--option=Dpkg::options::=--force-unsafe-io\\\', \\\'--assume-yes\\\', \\\'--quiet\\\', \\\'install\\\', \\\'chrony\\\']\\\\nExit code: 100\\\\nReason: -\\\\nStdout: -\\\\nStderr: -",))\'] were encountered in stage modules-config\n', 'module': 'tests.cloud_tests.testcases.modules.ntp_chrony', 'function': 'test_no_stages_errors', 'class': 'TestNtpChrony', 'error': 'AssertionError: 1 != 0 : errors [\'(\\\'ntp\\\', ProcessExecutionError("Unexpected error while running command.\\\\nCommand: [\\\'eatmydata\\\', \\\'apt-get\\\', \\\'--option=Dpkg::Options::=--force-confold\\\', \\\'--option=Dpkg::options::=--force-unsafe-io\\\', \\\'--assume-yes\\\', \\\'--quiet\\\', \\\'install\\\', \\\'chrony\\\']\\\\nExit code: 100\\\\nReason: -\\\\nStdout: -\\\\nStderr: -",))\'] were encountered in stage modules-config'}, {'traceback': 'Traceback (most recent call last):\n File "/var/lib/jenkins/slaves/torkoal/workspace/cloud-init-integration-lxd-a/cloud-init/tests/cloud_tests/testcases/base.py", line 149, in test_no_warnings_in_log\n [], warnings, msg="\'WARN\' found inside cloud-init.log")\nAssertionError: Lists differ: [] != ["2018-06-12 12:11:09,883 - util.py[WARNIN[128 chars]led"]\n\nSecond list contains 1 additional elements.\nFirst extra element 0:\n"2018-06-12 12:11:09,883 - util.py[WARNING]: Running module ntp (<module \'cloudinit.config.cc_ntp\' from \'/usr/lib/python3/dist-packages/cloudinit/config/cc_ntp.py\'>) failed"\n\n- []\n+ [\'2018-06-12 12:11:09,883 - util.py[WARNING]: Running module ntp (<module \'\n+ "\'cloudinit.config.cc_ntp\' from "\n+ "\'/usr/lib/python3/dist-packages/cloudinit/config/cc_ntp.py\'>) failed"] : \'WARN\' found inside cloud-init.log\n', 'module': 'tests.cloud_tests.testcases.modules.ntp_chrony', 'function': 'test_no_warnings_in_log', 'class': 'TestNtpChrony', 'error': '+ "\'/usr/lib/python3/dist-packages/cloudinit/config/cc_ntp.py\'>) failed"] : \'WARN\' found inside cloud-init.log'}]}, 'modules/ntp_pools': {'passed': True, 'failures': []}, 'modules/ssh_auth_key_fingerprints_disable': {'passed': True, 'failures': []}, 'modules/apt_configure_security': {'passed': True, 'failures': []}, 'modules/set_hostname': {'passed': True, 'failures': []}, 'modules/set_password_list': {'passed': True, 'failures': []}, 'modules/apt_configure_sources_ppa': {'passed': True, 'failures': []}, 'modules/set_hostname_fqdn': {'passed': True, 'failures': []}, 'modules/locale': {'passed': True, 'failures': []}, 'modules/ca_certs': {'passed': True, 'failures': []}, 'modules/apt_configure_primary': {'passed': True, 'failures': []}, 'modules/package_update_upgrade_install': {'passed': True, 'failures': []}, 'modules/debug_disable': {'passed': True, 'failures': []}, 'modules/final_message': {'passed': True, 'failures': []}, 'modules/apt_pipelining_os': {'passed': True, 'failures': []}, 'modules/lxd_bridge': {'passed': True, 'failures': []}, 'modules/ntp_timesyncd': {'passed': True, 'failures': []}, 'modules/snap': {'passed': True, 'failures': []}, 'modules/debug_enable': {'passed': True, 'failures': []}, 'modules/apt_configure_sources_list': {'passed': True, 'failures': []}, 'bugs/lp1628337': {'passed': True, 'failures': []}, 'modules/apt_configure_disable_suites': {'passed': True, 'failures': []}, 'modules/apt_configure_sources_keyserver': {'passed': True, 'failures': []}, 'modules/byobu': {'passed': True, 'failures': []}, 'modules/set_password_list_string': {'passed': True, 'failures': []}, 'modules/apt_configure_conf': {'passed': True, 'failures': []}, 'modules/ssh_keys_generate': {'passed': True, 'failures': []}, 'modules/ntp': {'passed': True, 'failures': []}}}}

verbose integration test output after this branch:
OK (skipped=2)
2018-06-12 18:47:07,118 - tests.cloud_tests - WARNING - test: platform='lxd', os='artful' failed 1 tests
2018-06-12 18:47:07,118 - tests.cloud_tests - DEBUG -
---- Verify summarized results:

Platform: lxd
  Distro: artful
    test modules passed:46 tests failed:2
    test failures:
    * tests.cloud_tests.testcases.modules.ntp_chrony.TestNtpChrony.test_no_stages_errors
          AssertionError: 1 != 0 : errors ['(\'ntp\', ProcessExecutionError("Unexpected error while running command.\\nCommand: [\'eatmydata\', \'apt-get\', \'--option=Dpkg::Options::=--force-confold\', \'--option=Dpkg::options::=--force-unsafe-io\', \'--assume-yes\', \'--quiet\', \'install\', \'chrony\']\\nExit code: 100\\nReason: -\\nStdout: -\\nStderr: -",))'] were encountered in stage modules-config
    * tests.cloud_tests.testcases.modules.ntp_chrony.TestNtpChrony.test_no_warnings_in_log
          + "'/usr/lib/python3/dist-packages/cloudinit/config/cc_ntp.py'>) failed"] : 'WARN' found inside cloud-init.log

[Ryan Harper (raharper)]

Nice!

Is there a reason for the indentation of the results?

Or can the "- Platform: lxd" be left justified?

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:ab9e431e4b40129f33e7747a1900551609dad899
https://jenkins.ubuntu.com/server/job/cloud-init-ci/75/
Executed test runs:
    SUCCESS: Checkout
    FAILED: Unit & Style Tests

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/75/rebuild

review: Needs Fixing (continuous-integration)

fa3014c... by Chad Smith on 2018-06-12

reduce indent on all summarized results

[Ryan Harper (raharper)]

Better. Approve with making ci happy.

review: Approve

[Joshua Powers (powersj)]

review: Approve

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:fa3014cd0cf3c1efb24f548ec51e9080fad42ddd
https://jenkins.ubuntu.com/server/job/cloud-init-ci/76/
Executed test runs:
    SUCCESS: Checkout
    FAILED: Unit & Style Tests

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/76/rebuild

review: Needs Fixing (continuous-integration)

e7917b9... by Chad Smith on 2018-06-12

pycodestyle fix unused variable

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:e7917b981b7eb762b0df7a73e42fcc9d7f850ecf
https://jenkins.ubuntu.com/server/job/cloud-init-ci/77/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/77/rebuild

review: Approve (continuous-integration)

[Chad Smith (chad.smith)]

An upstream commit landed for this bug.

To view that commit see the following URL:
https://git.launchpad.net/cloud-init/commit/?id=d0f6c460

diff --git a/tests/cloud_tests/verify.py b/tests/cloud_tests/verify.py
index 5a68a48..bfb2744 100644
--- a/tests/cloud_tests/verify.py
+++ b/tests/cloud_tests/verify.py
@@ -56,6 +56,51 @@ def verify_data(data_dir, platform, os_name, tests):
     return res
 
 
+def format_test_failures(test_result):
+    """Return a human-readable printable format of test failures."""
+    if not test_result['failures']:
+        return ''
+    failure_hdr = '    test failures:'
+    failure_fmt = '    * {module}.{class}.{function}\n          {error}'
+    output = []
+    for failure in test_result['failures']:
+        if not output:
+            output = [failure_hdr]
+        output.append(failure_fmt.format(**failure))
+    return '\n'.join(output)
+
+
+def format_results(res):
+    """Return human-readable results as a string"""
+    platform_hdr = 'Platform: {platform}'
+    distro_hdr = '  Distro: {distro}'
+    distro_summary_fmt = (
+        '    test modules passed:{passed} tests failed:{failed}')
+    output = ['']
+    counts = {}
+    for platform, platform_data in res.items():
+        output.append(platform_hdr.format(platform=platform))
+        counts[platform] = {}
+        for distro, distro_data in platform_data.items():
+            distro_failure_output = []
+            output.append(distro_hdr.format(distro=distro))
+            counts[platform][distro] = {'passed': 0, 'failed': 0}
+            for _, test_result in distro_data.items():
+                if test_result['passed']:
+                    counts[platform][distro]['passed'] += 1
+                else:
+                    counts[platform][distro]['failed'] += len(
+                        test_result['failures'])
+                    failure_output = format_test_failures(test_result)
+                    if failure_output:
+                        distro_failure_output.append(failure_output)
+            output.append(
+                distro_summary_fmt.format(**counts[platform][distro]))
+            if distro_failure_output:
+                output.extend(distro_failure_output)
+    return '\n'.join(output)
+
+
 def verify(args):
     """Verify test data.
 
@@ -90,7 +135,7 @@ def verify(args):
             failed += len(fail_list)
 
     # dump results
-    LOG.debug('verify results: %s', res)
+    LOG.debug('\n---- Verify summarized results:\n%s', format_results(res))
     if args.result:
         util.merge_results({'verify': res}, args.result)