Merge lp:~1chb1n/ubuntu-openstack-ci/mp-deploy into lp:ubuntu-openstack-ci

[Ryan Beisner]

unleash mp commentator

[Ryan Beisner (1chb1n)]

review: Approve

=== added file 'admin/820-tarmac-credentials-distribute.sh'
--- admin/820-tarmac-credentials-distribute.sh	1970-01-01 00:00:00 +0000
+++ admin/820-tarmac-credentials-distribute.sh	2014-10-17 19:16:56 +0000
@@ -0,0 +1,32 @@
+#!/bin/bash -e
+echo $0
+
+# Copy launchpad credentials file to all nodes
+# Requires that you have placed credentials in the populate dir
+
+. $(dirname $0)/../env-common
+
+f_name="credentials"
+f_dst_dir="${JENKINS_HOME}/.config/tarmac"
+f_dst="${f_dst_dir}/${f_name}"
+f_tmp="${U_HOME}/${f_name}"
+f_src_dir="../populate"
+f_src="${f_src_dir}/${f_name}"
+
+if [ ! -f "$f_src" ]; then
+  echo "Local source file ${f_src} not found."
+  pwd
+  exit 1
+fi
+
+for unit in $UNITS; do
+  echo "making tarmac config directory on ${unit}!"
+  juju ssh $unit "sudo su - jenkins -c 'mkdir -p /var/lib/jenkins/.config/tarmac'"
+  echo "copying ${f_src} into ${unit}"
+  juju scp $f_src ${unit}:${f_tmp}
+  echo "adjusting permissions"
+  juju ssh $unit "sudo mv $f_tmp $f_dst &&\
+    sudo chown jenkins:jenkins -R ${f_dst} &&\
+    sudo chmod 600 ${f_dst}"
+done
+

=== modified file 'job-parts/amulet_test.sh'
--- job-parts/amulet_test.sh	2014-10-09 20:54:32 +0000
+++ job-parts/amulet_test.sh	2014-10-17 19:16:56 +0000
@@ -1,11 +1,13 @@
 #!/bin/bash -e
 
+touch $WORKSPACE/try-amul
+
 env | grep -v "PASSWORD" | sort 2>&1 | tee env.$BUILD_NUMBER
 
 # Make sure expected env vars exist
 if [[ -z "$BASE_NAME" ]] || [[ -z "$BRANCH" ]]; then
   echo "No BRANCH and/or BASE_NAME env vars."
-  touch $WORKSPACE/fyi.env-vars.bad
+  touch $WORKSPACE/fyi-amul.env-vars.bad
 else
   # env vars are ok
   mkdir -pv $JENKINS_HOME/checkout || true
@@ -27,7 +29,7 @@
 
   if [[ ! -f $MAKEFILE ]]; then
     echo "No Makefile found."
-    touch $WORKSPACE/fyi.makefile.missing
+    touch $WORKSPACE/fyi-amul.makefile.missing
   else
     # makefile exists
     cd $CO_DIR
@@ -38,13 +40,13 @@
 
     if [[ -z $has_amulet ]]; then
       echo "No amulet test detected in Makefile."
-      touch $WORKSPACE/fyi.amulet-test.missing
+      touch $WORKSPACE/fyi-amul.amulet-test.missing
     else
       # amulet test
       make test 2>&1 | tee $WORKSPACE/make-test.$BUILD_NUMBER
       if [[ "${PIPESTATUS[0]}" != "0" ]]; then
         echo "Amulet test failed."
-        touch $WORKSPACE/fyi.amulet-test.failed
+        touch $WORKSPACE/fyi-amul.amulet-test.failed
       fi
     fi
   fi
@@ -53,10 +55,17 @@
 # Set build description
 ${OSCI_ROOT}/job-parts/b_d.sh || true
 
+# Post mp comment if triggered by a merge proposal
+if [[ -n "$MP_TRIGGER" ]]; then
+  # Dry run, debug enabled, ftm.
+  cd $OSCI_ROOT
+  ./mp_comment.py || true
+fi
+
 # Check for failures
 if [[ "$(ls $WORKSPACE -1)" == *.failed* ]] || \
    [[ "$(ls $WORKSPACE -1)" == *.missing* ]] || \
    [[ "$(ls $WORKSPACE -1)" == *.bad* ]]; then
-  ls $WORKSPACE -1
+  echo "Exiting 1"
   exit 1
 fi

=== modified file 'job-parts/lint_check.sh'
--- job-parts/lint_check.sh	2014-10-17 14:15:17 +0000
+++ job-parts/lint_check.sh	2014-10-17 19:16:56 +0000
@@ -1,11 +1,13 @@
 #!/bin/bash -e
 
+touch $WORKSPACE/try-lint
+
 env | grep -v "PASSWORD" | sort 2>&1 | tee env.$BUILD_NUMBER
 
 # Make sure expected env vars exist
 if [[ -z "$BASE_NAME" ]] || [[ -z "$BRANCH" ]]; then
   echo "No BRANCH and/or BASE_NAME env vars."
-  touch $WORKSPACE.fyi.env-vars.bad
+  touch $WORKSPACE.fyi-lint.env-vars.bad
 else
   # env vars are ok
   mkdir -pv $JENKINS_HOME/checkout || true
@@ -27,7 +29,7 @@
 
   if [[ ! -f $MAKEFILE ]]; then
     echo "No Makefile found."
-    touch $WORKSPACE/fyi.makefile.missing
+    touch $WORKSPACE/fyi-lint.makefile.missing
   else
     # makefile exists
     cd $CO_DIR
@@ -38,13 +40,13 @@
 
     if [[ -z $has_flake ]]; then
       echo "No lint test detected in Makefile."
-      touch $WORKSPACE/fyi.lint-check.missing
+      touch $WORKSPACE/fyi-lint.lint-check.missing
     else
       # lint check
       make lint 2>&1 | tee $WORKSPACE/make-lint.$BUILD_NUMBER
       if [[ "${PIPESTATUS[0]}" != "0" ]]; then
         echo "Lint check test failed."
-        touch $WORKSPACE/fyi.lint-check.failed
+        touch $WORKSPACE/fyi-lint.lint-check.failed
       fi
     fi
   fi
@@ -57,13 +59,13 @@
 if [[ -n "$MP_TRIGGER" ]]; then
   # Dry run, debug enabled, ftm.
   cd $OSCI_ROOT
-  ./mp_comment.py -0d || true
+  ./mp_comment.py || true
 fi
 
 # Check for failures
 if [[ "$(ls $WORKSPACE -1)" == *.failed* ]] || \
    [[ "$(ls $WORKSPACE -1)" == *.missing* ]] || \
    [[ "$(ls $WORKSPACE -1)" == *.bad* ]]; then
-  ls $WORKSPACE -1
+  echo "Exiting 1"
   exit 1
 fi

=== modified file 'job-parts/trigger_by_mp_scanner.sh'
--- job-parts/trigger_by_mp_scanner.sh	2014-10-10 06:07:20 +0000
+++ job-parts/trigger_by_mp_scanner.sh	2014-10-17 19:16:56 +0000
@@ -10,19 +10,19 @@
     ./scan_merge_props.py -0 -d
     ;;
   FORCE_AMULET_ONLY)
-    ./scan_merge_props.py -F -N --XD --XL --XU
+    ./scan_merge_props.py -F -N --XD --XL --XU -d
     ;;
   FORCE_UNIT_ONLY)
-    ./scan_merge_props.py -F -N --XD --XL --XA
+    ./scan_merge_props.py -F -N --XD --XL --XA -d
     ;;
   FORCE_DEPLOY_ONLY)
-    ./scan_merge_props.py -F -N --XA --XL --XU
+    ./scan_merge_props.py -F -N --XA --XL --XU -d
     ;;
   FORCE_LINT_ONLY)
-    ./scan_merge_props.py -F -N --XD --XA --XU
+    ./scan_merge_props.py -F -N --XD --XA --XU -d
     ;;
   FORCE_ALL)
-    ./scan_merge_props.py -F -N
+    ./scan_merge_props.py -F -N -d
     ;;
   FORCE_ALL_DRY_RUN)
     ./scan_merge_props.py -F0d

=== modified file 'job-parts/unit_test.sh'
--- job-parts/unit_test.sh	2014-10-09 20:54:32 +0000
+++ job-parts/unit_test.sh	2014-10-17 19:16:56 +0000
@@ -1,11 +1,13 @@
 #!/bin/bash -e
 
+touch $WORKSPACE/try-unit
+
 env | grep -v "PASSWORD" | sort 2>&1 | tee env.$BUILD_NUMBER
 
 # Make sure expected env vars exist
 if [[ -z "$BASE_NAME" ]] || [[ -z "$BRANCH" ]]; then
   echo "No BRANCH and/or BASE_NAME env vars."
-  touch $WORKSPACE/fyi.env-vars.bad
+  touch $WORKSPACE/fyi-unit.env-vars.bad
 else
   # env vars are ok
   mkdir -pv $JENKINS_HOME/checkout || true
@@ -27,7 +29,7 @@
 
   if [[ ! -f $MAKEFILE ]]; then
     echo "No Makefile found."
-    touch $WORKSPACE/fyi.makefile.missing
+    touch $WORKSPACE/fyi-unit.makefile.missing
   else
     # makefile exists
     cd $CO_DIR
@@ -39,7 +41,7 @@
 
     if [[ -z "$nose_in_unit_test" ]] && [[ -z "$nose_in_test" ]]; then
       echo "No unit test detected in Makefile."
-      touch $WORKSPACE/fyi.unit-test.missing
+      touch $WORKSPACE/fyi-unit.unit-test.missing
     else
       # unit test
       if [[ -n "$nose_in_unit_test" ]]; then
@@ -51,7 +53,7 @@
       fi
       if [[ "${PIPESTATUS[0]}" != "0" ]]; then
         echo "Unit test failed."
-        touch $WORKSPACE/fyi.unit-test.failed
+        touch $WORKSPACE/fyi-unit.unit-test.failed
       fi
     fi
   fi
@@ -61,10 +63,17 @@
 # Set build description
 ${OSCI_ROOT}/job-parts/b_d.sh || true
 
+# Post mp comment if triggered by a merge proposal
+if [[ -n "$MP_TRIGGER" ]]; then
+  # Dry run, debug enabled, ftm.
+  cd $OSCI_ROOT
+  ./mp_comment.py || true
+fi
+
 # Check for failures
 if [[ "$(ls $WORKSPACE -1)" == *.failed* ]] || \
    [[ "$(ls $WORKSPACE -1)" == *.missing* ]] || \
    [[ "$(ls $WORKSPACE -1)" == *.bad* ]]; then
-  ls $WORKSPACE -1
+  echo "Exiting 1"
   exit 1
 fi

=== modified file 'mp_comment.py'
--- mp_comment.py	2014-10-16 22:22:32 +0000
+++ mp_comment.py	2014-10-17 19:16:56 +0000
@@ -16,18 +16,20 @@
 USAGE = '''Usage: %prog [options]
 
 Default behavior:
-  Constructs merge proposal message and posts it as a comment on the MP.
+  Constructs merge proposal message by examining WORKSPACE files.  Posts
+  comment on the merge proposal.
 
 For example:
-
-    %prog
+    %prog -0d
+  Dry run with debug output.  Displays comment but does not save or post it.
+
+    %prog -Xd
+  Dry run with debug output.  Creates comment file but does not post it.
+
+    %prog -q
+  Less noisy output.  Create message, comment file, and post it.
 '''
 
-LLIMIT = 25
-TTYPES = {'lint': 'make-lint.*',
-          'unit': 'unit-test.*',
-          'amulet': 'make-test.*'
-          }
 
 try:
     WORKSPACE = os.environ['WORKSPACE']
@@ -54,13 +56,19 @@
     JOB_NAME = 'manual-job'
 
 
-def create_mp_comment_file(contexts, source, out_file):
+def create_mp_comment_file(contexts, source, out_file, params):
     '''Create MP comment file, return comment filename
     '''
+    (opts, args) = params
+
+    if opts.dry:
+        out_file = tempfile.mktemp()
+        logging.info('Dry run, rendering to {} instead'.format(out_file))
+
     templates_dir = 'populate'
     for idx, context in enumerate(contexts):
         target = os.path.join(WORKSPACE, out_file.format(idx))
-        logging.debug('Rendering file: {}'.format(target))
+        logging.info('Rendering file: {}'.format(target))
         osci_utils.render(source, target, context, templates_dir)
     return target
 
@@ -74,9 +82,12 @@
                       help="Enable debug logging",
                       dest="debug", action="store_true", default=False)
     parser.add_option('-0', '--dry-run',
-                      help='Construct and display the message, but do not '
+                      help='Construct and display the message, but do not'
                            'create comment file and do not post comment.',
                       dest='dry', action='store_true', default=False)
+    parser.add_option('-X', '--no-post',
+                      help='Do not post comment.',
+                      dest='no_post', action='store_true', default=False)
     parser.add_option('-q', '--quiet',
                       help='Do not display the message to console.',
                       dest='quiet', action='store_true', default=False)
@@ -96,6 +107,8 @@
 
     logging.info('Workspace dir: {}'.format(WORKSPACE))
 
+    conf_map = osci_utils.read_yaml(osci_utils.MAP_FILE)
+
     # gather info and results for comment
     logging.info('Gathering env var items')
     contexts = []
@@ -110,71 +123,126 @@
         'JOB_NAME': JOB_NAME
     })
 
+    # construct results blurb by examining artifacts
     logging.info('Constructing results blurb')
+    summary = []
     results = []
-    fyis = glob(os.path.join(WORKSPACE, 'fyi.*.*'))
-    if len(fyis) > 0:
-        for fyi in fyis:
-            _line = '    FYI: {}'.format(
-                os.path.split(fyi)[1][4:].replace('.', ' '))
-            logging.debug(_line)
-            results.append(_line)
-    else:
-        logging.info('No "fyi" files found to include.')
-
-    for ttype, glob_str in TTYPES.iteritems():
-        res_files = glob(os.path.join(WORKSPACE, glob_str))
-        for res_file in res_files:
-            _line = '    Test type: {}\n'.format(ttype)
-            logging.debug(_line)
-            results.append(_line)
-            _line = ('Results (limited to the last '
-                     '{} lines) from {}:'.format(LLIMIT, res_file))
-            logging.debug(_line)
-            results.append(_line)
-            with open(res_file) as _file:
-                res_lines = [f_line.rstrip() for f_line in _file][-LLIMIT:]
-            logging.debug(res_lines)
-            results.extend(res_lines)
-
-    contexts[0]['results_summary'] = '\n'.join(results)
-
-    logging.debug('contexts: {}'.format(contexts))
+    something_ran = False
+    for trigger_test, tt_attrs in conf_map['mp-trigger-tests'].iteritems():
+        # look for try touchfiles
+        try_files = []
+        logging.debug('Checking for {} try files'.format(trigger_test))
+        for try_masq in tt_attrs['try-masq']:
+            these_try_files = glob(os.path.join(WORKSPACE, try_masq))
+            try_files.extend(these_try_files)
+        if len(try_files) > 0:
+            something_ran = True
+            logging.info('Found {} try file(s)'.format(trigger_test))
+        else:
+            # did not run
+            logging.info('No {} try file(s) found'.format(trigger_test))
+            continue
+        # look for result files
+        res_files = []
+        logging.debug('Checking result masq for {}'.format(trigger_test))
+        for res_masq in tt_attrs['result-masq']:
+            these_res_files = glob(os.path.join(WORKSPACE, res_masq))
+            res_files.extend(these_res_files)
+        if len(res_files) > 0:
+            logging.info('Found {} results file(s)'.format(trigger_test))
+            # handle results files
+            for res_file in res_files:
+                llimit = conf_map['mp-conf']['line-limit']
+                _line = ('\n{} Results (max last {} lines) from \n{}:'.format(
+                    trigger_test.upper(), llimit, res_file))
+                results.append(_line)
+                with open(res_file) as _file:
+                    res_lines = [f_line.rstrip() for f_line in _file][-llimit:]
+                results.extend(res_lines)
+        else:
+            _line = ('\n{} Results not found.'.format(trigger_test.upper()))
+            results.append(_line)
+            logging.info('No {} results files found to '
+                         'include'.format(trigger_test))
+        # look for fail touchfiles
+        logging.debug('Checking for {} fyi files'.format(trigger_test))
+        passed = True
+        for fail_masq in tt_attrs['fail-masq']:
+            logging.debug('Checking fail masq: {}'.format(fail_masq))
+            fyis = glob(os.path.join(WORKSPACE, fail_masq))
+            if len(fyis) > 0:
+                passed = False
+                logging.debug('Found {} fyi file(s)'.format(trigger_test))
+                # handle fyi files
+                for fyi in fyis:
+                    _line = '    {} FAIL: {}'.format(
+                        trigger_test.upper(),
+                        os.path.split(fyi)[1][8:].replace('.', ' '))
+                    logging.debug(_line)
+                    summary.append(_line)
+            else:
+                logging.info('No "fyi" files found to '
+                             'include {}'.format(fail_masq))
+        # ran and didn't fail
+        if passed:
+            _line = ('    {} OK:  believed to pass, but you should '
+                     'confirm results'.format(trigger_test.upper()))
+            summary.append(_line)
+
+    if not something_ran:
+        logging.error('It appears that nothing ran here.')
+        sys.exit(1)
+
+    # add summary and results to context for template
+    contexts[0]['results_summary'] = '\n'.join(summary)
+    contexts[0]['results_summary'] += '\n'
+    contexts[0]['results_summary'] += '\n'.join(results)
 
     # create mp comment file
     logging.info('Creating mp-comment file')
     comment_filename = create_mp_comment_file(contexts,
                                               source='mp-comment.tmpl',
-                                              out_file='mp-comment.{}')
+                                              out_file='mp-comment.{}',
+                                              params=params)
 
-    # read mp comment file
+    # read mp comment file and display to console
+    subject = '[UOSCI] test result for {}'.format(DISPLAY_NAME)
+    logging.debug('Subject: {}'.format(subject))
     f = open(comment_filename, 'r')
     comment = f.read()
     f.close()
-
     if not opts.quiet:
         logging.info('Comment:\n')
+        print '-' * 80
+        print subject
         print comment
+        print '-' * 80
+
+    # check for sandbox override (a noise avoider during dev)
+    if conf_map['mp-conf']['mp-comment-sandbox']:
+        lp_branch_url = conf_map['mp-conf']['sandbox-lp-branch-url']
+        mp_url = conf_map['mp-conf']['sandbox-mp-url']
+        logging.info('Sandbox is ENABLED: comments, if '
+                     'enabled, will hit {} instead'.format(mp_url))
 
     # post mp comment
-    if not opts.dry:
-        logging.debug('Connecting to Launchpad API')
-        launchpad = osci_utils.lpl_connect(anonymous=False)
-
-        lp_branch_url = "lp:ubuntu-openstack-ci"
-        mp_url = 'https://code.launchpad.net/~1chb1n/ubuntu-openstack-ci/mptest0/+merge/238615'
-
-        logging.debug('Looking up MP object from LP API')
-        mp = osci_utils.lpl_get_mp(launchpad, mp_url, lp_branch_url)
-        if mp:
+    logging.debug('Connecting to LP via API')
+    launchpad = osci_utils.lpl_connect(anonymous=False)
+
+    logging.debug('Looking up MP object from LP API')
+    mp = osci_utils.lpl_get_mp(launchpad, mp_url, lp_branch_url)
+    if mp:
+        if not opts.dry and not opts.no_post:
             logging.info('Posting comment to {}'.format(mp_url))
-            subject = '[UOSCI] test result for {}'.format(DISPLAY_NAME)
             osci_utils.lpl_mp_post_comment(launchpad, mp, subject, comment)
         else:
-            logging.error('Merge proposal not found for '
-                          '{} on {}'.format(mp_url, lp_branch_url))
-            sys.exit(1)
+            logging.info('Not posting comment due to options')
+    else:
+        logging.error('Merge proposal not found for '
+                      '{} on {}'.format(mp_url, lp_branch_url))
+        sys.exit(1)
 
 
 if __name__ == '__main__':
+    print 'Ubuntu OSCI Merge Proposal Commentator'
     main()

=== modified file 'populate/mappings.yaml'
--- populate/mappings.yaml	2014-10-10 15:16:27 +0000
+++ populate/mappings.yaml	2014-10-17 19:16:56 +0000
@@ -13,17 +13,24 @@
   state-file: 'branch-state'
   mp-state-file: 'mp-state'
 
+mp-conf:
+  line-limit: 25
+# post ALL MP comments to this alternate MP when mp-comment-sandbox is True
+# ie. contain the noise during osci dev
+  mp-comment-sandbox: False
+  sandbox-lp-branch-url: 'lp:ubuntu-openstack-ci'
+  sandbox-mp-url: 'https://code.launchpad.net/~1chb1n/ubuntu-openstack-ci/mptest0/+merge/238615'
+
 # merge proposal status to trigger tests
 mp-status-filter:
- - "Needs review"
-# - "Work in progress"
-# - "Approved"
-# - "Needs review"
-# - "Rejected"
-# - "Merged"
-# - "Code failed to merge"
-# - "Queued"
-# - "Superseded"
+ - 'Needs review'
+# - 'Work in progress'
+# - 'Approved'
+# - 'Rejected'
+# - 'Merged'
+# - 'Code failed to merge'
+# - 'Queued'
+# - 'Superseded'
 
 # the openstack version which shipped with an ubuntu release
 # not used by OSCI code yet
@@ -105,3 +112,53 @@
 #  - trusty-juno-staging
   - utopic-juno
 #  - utopic-juno-proposed
+
+mp-trigger-tests:
+  lint:
+    order: 100
+    funcs:
+    - 'run_lint_test'
+    try-masq:
+    - 'try-lint*'
+    result-masq:
+    - 'make-lint.*'
+    fail-masq:
+    - 'fyi-lint.*.failed'
+    - 'fyi-lint.*.missing'
+  unit:
+    order: 120
+    funcs:
+    - 'run_un_test'
+    try-masq:
+    - 'try-unit*'
+    result-masq:
+    - 'unit-test.*'
+    fail-masq:
+    - 'fyi-unit.*.failed'
+    - 'fyi-unit.*.missing'
+  amulet:
+    order: 180
+    funcs:
+    - 'run_amulet_test'
+    try-masq:
+    - 'try-amul*'
+    result-masq:
+    - 'make-test.*'
+    fail-masq:
+    - 'fyi-amul.*.failed'
+    - 'fyi-amul.*.missing'
+  deploy:
+    order: 199
+    funcs:
+    - 'run_deploy_test'
+    try-masq:
+    - 'try-depl*'
+    result-masq:
+    - 'deployer.*'
+    - 'configure.*'
+    - 'tempstack-configure.*'
+    - 'tempest.*'
+    fail-masq:
+    - 'fyi-depl.*.failed'
+    - 'fyi-depl.*.missing'
+#    - 'fyi-depl.*.bad'

=== modified file 'populate/mp-comment.tmpl'
--- populate/mp-comment.tmpl	2014-10-16 22:14:46 +0000
+++ populate/mp-comment.tmpl	2014-10-17 19:16:56 +0000
@@ -1,15 +1,20 @@
+UOSCI bot says:
+This MP triggered a test on the Ubuntu OSCI system.  Here is a summary of results.
+
+{{BUILD_DISPLAY_NAME}}
 {{JOB_NAME}}
-This MP triggered a test on the Ubuntu OSCI system ({{JENKINS_URL}}), which is currently on a Canonical private network while it is in development.  We wanted you to have feedback now, so here is a summary of a test which was triggered by this MP.  Tests are triggered if the proposed branch revno/revid changes, or if this MP is placed into "Needs review" status after being otherwise for > 1hr.  Human review of results is recommended.
-
-{{DISPLAY_NAME}}
-    MP URL: {{MP_TRIGGER}}
-    Build name: {{BUILD_DISPLAY_NAME}}
-    Proposed branch: {{MP_SRC_BR}}
-
-Additional details and artifacts are available internally:
-    {{BUILD_URL}}
+
+This build:
+{{BUILD_URL}}
+
+MP URL:
+{{MP_TRIGGER}}
+
+Proposed branch:
+{{MP_SRC_BR}}
 
 Results summary:
 {{results_summary}}
 
-{{BUILD_URL}}
+Ubuntu OSCI Jenkins is currently in development on a Canonical private network, but we plan to publish results to a public instance soon.  Tests are triggered if the proposed branch rev changes, or if the MP is placed into "Needs review" status after being otherwise for >= 1hr.  Human review of results is still recommended.
+{{JENKINS_URL}}