Merge ~saviq/jenkaas-jobs:use-upstream-matrix-trigger into jenkaas-jobs:master

[Michał Sawicz]

Drop raw matrix-trigger

Cherry-picked jenkins-job-builder has native support for it.

NOTE: You can now drop the repeated, escaped autopkgtest_filter, see
changes in project-example.yaml

FIXME--! Thanks Olivier for nudging me in this direction ;)

[Olivier Tilloy (osomon)]

LGTM, thanks for that!
I’ve deployed to the system-apps jenkaas instance and everything seems to work fine.

review: Approve

diff --git a/README.rst b/README.rst
index 8c0e9a1..2a5a9ea 100644
--- a/README.rst
+++ b/README.rst
@@ -53,18 +53,40 @@ Documentation <http://docs.openstack.org/infra/jenkins-job-builder/>`_. The
 example-project.yaml file has comments that should help you set things for your
 project just-so.
 
+Using jenkaas-jobs
+------------------
+You'll need to use a virtualenv for all the dependencies to be set up in a
+predictable way:
+
+.. code-block::
+
+ $ virtualenv -p python3 .venv
+ $ source .venv/bin/activate
+ (.venv)$ pip install -r requirements.txt
+
+To get out of the virtualenv:
+
+.. code-block::
+ (.venv)$ deactivate
+
+To run jenkins-jobs:
+
+.. code-block::
+ $ source .venv/bin/activate
+ (.venv)$ jenkins-jobs
+
 Testing your job definitions
 ----------------------------
 .. code-block::
 
- $ jenkins-job-builder --conf your-project.conf test \
+ (.venv)$ jenkins-jobs --conf your-project.conf test \
      your-project.yaml:jenkaas-jobs/jenkaas_jobs/yaml --output test
 
 Creating/Updating your job definitions
 --------------------------------------
 .. code-block::
 
- $ jenkins-job-builder --conf your-project.conf update \
+ (.venv)$ jenkins-jobs --conf your-project.conf update \
      your-project.yaml:jenkaas-jobs/jenkaas_jobs/yaml
 
 Extending
diff --git a/jenkaas_jobs/__init__.py b/jenkaas_jobs/__init__.py
index 81ad8c9..5ca2982 100644
--- a/jenkaas_jobs/__init__.py
+++ b/jenkaas_jobs/__init__.py
@@ -44,6 +44,7 @@ jenkins_rules = {
         "project",
         "matrix-project",
         "properties",
+        "configs",
         "hudson.model.StringParameterDefinition",
         "hudson.model.RunParameterDefinition",
         "hudson.tasks.ArtifactArchiver",
diff --git a/jenkaas_jobs/yaml/launchpad/lp-autoland.yaml b/jenkaas_jobs/yaml/launchpad/lp-autoland.yaml
index 34df322..0110145 100644
--- a/jenkaas_jobs/yaml/launchpad/lp-autoland.yaml
+++ b/jenkaas_jobs/yaml/launchpad/lp-autoland.yaml
@@ -32,14 +32,15 @@
             source_branch=${{landing_candidate}}
             source_revision=${{candidate_revision}}
             target_branch={branch}
-    - matrix-trigger:
-        job: test-0-autopkgtest
-        parameters: |
-          build_run=build#${{TRIGGERED_BUILD_NUMBER_build}}
-          testnames={autopkgtest_names}
-        filter: '{autopkgtest_filter}'
-        raw_parameters: |
-          <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
+    - trigger-builds:
+        - project:
+          - test-0-autopkgtest
+          block: true
+          current-parameters: true
+          predefined-parameters: |
+            build_run=build#${{TRIGGERED_BUILD_NUMBER_build}}
+            testnames={autopkgtest_names}
+          restrict-matrix-project: '{autopkgtest_filter}'
     wrappers:
     - timestamps
     - build-name:
diff --git a/jenkaas_jobs/yaml/launchpad/lp-ci.yaml b/jenkaas_jobs/yaml/launchpad/lp-ci.yaml
index 2e72df5..5a8f05d 100644
--- a/jenkaas_jobs/yaml/launchpad/lp-ci.yaml
+++ b/jenkaas_jobs/yaml/launchpad/lp-ci.yaml
@@ -32,14 +32,15 @@
             source_branch=${{landing_candidate}}
             source_revision=${{candidate_revision}}
             target_branch={branch}
-    - matrix-trigger:
-        job: test-0-autopkgtest
-        parameters: |
-          build_run=build#${{TRIGGERED_BUILD_NUMBER_build}}
-          testnames={autopkgtest_names}
-        filter: '{autopkgtest_filter}'
-        raw_parameters: |
-          <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
+    - trigger-builds:
+        - project:
+          - test-0-autopkgtest
+          block: true
+          current-parameters: true
+          predefined-parameters: |
+            build_run=build#${{TRIGGERED_BUILD_NUMBER_build}}
+            testnames={autopkgtest_names}
+          restrict-matrix-project: '{autopkgtest_filter}'
     wrappers:
     - timestamps
     - build-name:
diff --git a/jenkaas_jobs/yaml/macros.yaml b/jenkaas_jobs/yaml/macros.yaml
index c9f8064..b6c724d 100644
--- a/jenkaas_jobs/yaml/macros.yaml
+++ b/jenkaas_jobs/yaml/macros.yaml
@@ -9,22 +9,14 @@
             <setForMatrix>{set_for_matrix}</setForMatrix>
           </hudson.plugins.descriptionsetter.DescriptionSetterPublisher>
 
-- builder: # FIXME: JJB can't do matrix combinations parameter on the trigger
-    name: matrix-trigger
-    builders:
-    - raw:
-        raw_parameters: ''
-        xml:
-          !include-raw: matrix-trigger.xml
-
 - builder:
     name: node-trigger
     builders:
-    - matrix-trigger:
-        job: '{job}'
-        filter: node=="${{node}}"
-        parameters: ''
-        raw_parameters: ''
+    - trigger-builds:
+      - project:
+        - '{job}'
+        block: true
+        restrict-matrix-project: node=="${{node}}"
 
 - parameter:
     name: extra_repositories_parameters
diff --git a/jenkaas_jobs/yaml/matrix-trigger.xml b/jenkaas_jobs/yaml/matrix-trigger.xml
 deleted file mode 100644
index 892c792..0000000
--- a/jenkaas_jobs/yaml/matrix-trigger.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<hudson.plugins.parameterizedtrigger.TriggerBuilder>
-  <configs>
-    <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
-      <configs>
-        <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-          <properties>{parameters}</properties>
-        </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-        <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-          <filter>{filter}</filter>
-        </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-        {raw_parameters}
-      </configs>
-      <projects>{job}</projects>
-      <condition>ALWAYS</condition>
-      <triggerWithNoParameters>false</triggerWithNoParameters>
-      <buildAllNodesWithLabel>false</buildAllNodesWithLabel>
-      <block>
-        <buildStepFailureThreshold>
-          <name>FAILURE</name>
-          <ordinal>2</ordinal>
-          <color>RED</color>
-          <completeBuild>true</completeBuild>
-        </buildStepFailureThreshold>
-        <unstableThreshold>
-          <name>UNSTABLE</name>
-          <ordinal>1</ordinal>
-          <color>YELLOW</color>
-          <completeBuild>true</completeBuild>
-        </unstableThreshold>
-        <failureThreshold>
-          <name>FAILURE</name>
-          <ordinal>2</ordinal>
-          <color>RED</color>
-          <completeBuild>true</completeBuild>
-        </failureThreshold>
-      </block>
-    </hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
-  </configs>
-</hudson.plugins.parameterizedtrigger.TriggerBuilder>
diff --git a/project-example.yaml b/project-example.yaml
index 443b61f..21cda2f 100644
--- a/project-example.yaml
+++ b/project-example.yaml
@@ -96,11 +96,8 @@
     # Which autopktests to run on which architectures.
     #
     # Available axes: label, release and testname
-    #
-    # FIXME: this has to use XML entities for & due to a Jenkins Job Builder
-    #        limitation.
     autopkgtest_filter: &autopkgtest_filter
-      (label=="amd64" &amp;&amp; testname=="test-name.sh") || (label=="phone-armhf" &amp;&amp; testname=="autopilot.sh" &amp;&amp; release=="vivid+overlay")
+      (label=="amd64" && testname=="test-name.sh") || (label=="phone-armhf" && testname=="autopilot.sh" && release=="vivid+overlay")
 
     # A list of Launchpad projects to configure for CI, each of which can have
     # overrides for the per-project variables above as key-value pairs.
@@ -135,10 +132,7 @@
     - build-jobs
     
     # Run tests on the built packages.
-    - test-jobs:
-        # FIXME: this is the same filter as above, but without entities. 
-        autopkgtest_filter:
-          (label=="amd64" && testname=="test-name.sh") || (label=="phone-armhf" && testname=="autopilot.sh" && release=="vivid+overlay")
+    - test-jobs
     
     # Generic jobs for interacting with launchpad.
     - launchpad-jobs
diff --git a/requirements.txt b/requirements.txt
index e7c9015..5317e3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
 pbr>=1.0.0,<2.0
+git+https://git.launchpad.net/~jenkaas-hackers/+git/jenkins-job-builder@5822a2fb
diff --git a/test-requirements.txt b/test-requirements.txt
index ff5d80a..9bf7549 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1 +1,3 @@
-jenkins-job-builder>=1.4
+nose
+nose-ignore-docstring
+ddt
diff --git a/tests/fixtures/launchpad/lp-four-autoland.xml b/tests/fixtures/launchpad/lp-four-autoland.xml
index 8413199..1c1e385 100644
--- a/tests/fixtures/launchpad/lp-four-autoland.xml
+++ b/tests/fixtures/launchpad/lp-four-autoland.xml
@@ -103,7 +103,7 @@ target_branch=lp:four
 testnames=</properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter/>
+              <filter>(label==&quot;amd64&quot;) &amp;&amp; (release==&quot;devel&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/launchpad/lp-four-ci.xml b/tests/fixtures/launchpad/lp-four-ci.xml
index 3953b46..18a9c78 100644
--- a/tests/fixtures/launchpad/lp-four-ci.xml
+++ b/tests/fixtures/launchpad/lp-four-ci.xml
@@ -104,7 +104,7 @@ target_branch=lp:four</properties>
 testnames=</properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter></filter>
+              <filter>(label==&quot;amd64&quot;) &amp;&amp; (release==&quot;devel&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/launchpad/lp-one-ci.xml b/tests/fixtures/launchpad/lp-one-ci.xml
index 2d940c3..9c71787 100644
--- a/tests/fixtures/launchpad/lp-one-ci.xml
+++ b/tests/fixtures/launchpad/lp-one-ci.xml
@@ -103,7 +103,7 @@ target_branch=lp:one</properties>
 testnames=</properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter></filter>
+              <filter>(label==&quot;amd64&quot;) &amp;&amp; (release==&quot;devel&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/launchpad/lp-three-autoland.xml b/tests/fixtures/launchpad/lp-three-autoland.xml
index c1b70f3..88f4934 100644
--- a/tests/fixtures/launchpad/lp-three-autoland.xml
+++ b/tests/fixtures/launchpad/lp-three-autoland.xml
@@ -103,7 +103,7 @@ testnames=test-three
 </properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter/>
+              <filter>(label==&quot;amd64&quot;) &amp;&amp; (release==&quot;devel&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/launchpad/lp-three-ci.xml b/tests/fixtures/launchpad/lp-three-ci.xml
index 7de1709..4c0bb36 100644
--- a/tests/fixtures/launchpad/lp-three-ci.xml
+++ b/tests/fixtures/launchpad/lp-three-ci.xml
@@ -103,7 +103,7 @@ target_branch=lp:three/devel</properties>
 testnames=</properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter></filter>
+              <filter>(label==&quot;amd64&quot;) &amp;&amp; (release==&quot;devel&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/launchpad/lp-two-autoland.xml b/tests/fixtures/launchpad/lp-two-autoland.xml
index e563d1f..7e7b52d 100644
--- a/tests/fixtures/launchpad/lp-two-autoland.xml
+++ b/tests/fixtures/launchpad/lp-two-autoland.xml
@@ -103,7 +103,7 @@ testnames=test-one test-two
 </properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter>(label=="armhf").implies(testname=="test-one")</filter>
+              <filter>(label==&quot;armhf&quot;) &amp;&amp; (release==&quot;stable&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/launchpad/lp-two-ci.xml b/tests/fixtures/launchpad/lp-two-ci.xml
index f79c7f2..d1b8a46 100644
--- a/tests/fixtures/launchpad/lp-two-ci.xml
+++ b/tests/fixtures/launchpad/lp-two-ci.xml
@@ -103,7 +103,7 @@ target_branch=lp:two</properties>
 testnames=test-one test-two</properties>
             </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
-              <filter>(label==&quot;armhf&quot;).implies(testname==&quot;test-one&quot;)</filter>
+              <filter>(label==&quot;armhf&quot;) &amp;&amp; (release==&quot;stable&quot;)</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
             <hudson.plugins.parameterizedtrigger.CurrentBuildParameters/>
           </configs>
diff --git a/tests/fixtures/maintain/maintain.xml b/tests/fixtures/maintain/maintain.xml
index 861a1eb..7fdcb33 100644
--- a/tests/fixtures/maintain/maintain.xml
+++ b/tests/fixtures/maintain/maintain.xml
@@ -55,9 +55,6 @@
       <configs>
         <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
           <configs>
-            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-              <properties></properties>
-            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
               <filter>node==&quot;${node}&quot;</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
@@ -93,9 +90,6 @@
       <configs>
         <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
           <configs>
-            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-              <properties></properties>
-            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
               <filter>node==&quot;${node}&quot;</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
diff --git a/tests/fixtures/prepare/prepare.xml b/tests/fixtures/prepare/prepare.xml
index 95677eb..0a53c3d 100644
--- a/tests/fixtures/prepare/prepare.xml
+++ b/tests/fixtures/prepare/prepare.xml
@@ -51,9 +51,6 @@
       <configs>
         <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
           <configs>
-            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-              <properties></properties>
-            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
               <filter>node==&quot;${node}&quot;</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
@@ -89,9 +86,6 @@
       <configs>
         <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
           <configs>
-            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-              <properties></properties>
-            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
               <filter>node==&quot;${node}&quot;</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
@@ -127,9 +121,6 @@
       <configs>
         <hudson.plugins.parameterizedtrigger.BlockableBuildTriggerConfig>
           <configs>
-            <hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
-              <properties></properties>
-            </hudson.plugins.parameterizedtrigger.PredefinedBuildParameters>
             <hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
               <filter>node==&quot;${node}&quot;</filter>
             </hudson.plugins.parameterizedtrigger.matrix.MatrixSubsetBuildParameters>
diff --git a/tests/fixtures/test/test-0-autopkgtest.xml b/tests/fixtures/test/test-0-autopkgtest.xml
index 4e4d993..8391283 100644
--- a/tests/fixtures/test/test-0-autopkgtest.xml
+++ b/tests/fixtures/test/test-0-autopkgtest.xml
@@ -27,6 +27,7 @@
         <hudson.plugins.matrix__configuration__parameter.MatrixCombinationsParameterDefinition plugin="matrix-combinations-parameter@1.0.9">
           <name>configurations</name>
           <description/>
+          <defaultCombinationFilter>(label==&quot;amd64&quot;) &amp;&amp; (release==&quot;devel&quot;)</defaultCombinationFilter>
         </hudson.plugins.matrix__configuration__parameter.MatrixCombinationsParameterDefinition>
         <hudson.model.StringParameterDefinition>
           <name>testnames</name>
diff --git a/tests/project.yaml b/tests/project.yaml
index ffc40c8..df730b4 100644
--- a/tests/project.yaml
+++ b/tests/project.yaml
@@ -19,7 +19,7 @@
     chroot_filter:
       (node=="jenkins-slave-2").implies(arch=="amd64" || arch=="i386") && (node=="cyclops-node02").implies(arch=="armhf")
     autopkgtest_filter: &autopkgtest_filter
-        '' 
+      (label=="amd64") && (release=="devel")
     extra_repositories: |
       deb http://ppa.launchpad.net/test-ci-team/appa RELEASE main
     extra_repository_keys: AA22CC44
@@ -27,7 +27,7 @@
     - one
     - two: &two
         autopkgtest_names: test-one test-two
-        autopkgtest_filter: (label==&quot;armhf&quot;).implies(testname==&quot;test-one&quot;)
+        autopkgtest_filter: (label=="armhf") && (release=="stable")
     - three: &three
         branch: lp:three/devel
     - four: &four