Merge lp:~zyga/linaro-python-dashboard-bundle/test-duration into lp:linaro-python-dashboard-bundle/linaro-11.11

[Zygmunt Krynicki]

This branch adds the 1.4 document format that can encode the test duration (duration of an entire test run)

[Michael Hudson-Doyle (mwhudson)]

Seems fine. I wonder if there's a neat way of seeing the difference between dashboard_bundle_format_1.3.json and dashboard_bundle_format_1.4.json? The text diff seems clear enough in this case though.

review: Approve

=== modified file 'doc/changes.rst'
--- doc/changes.rst	2012-04-02 16:39:49 +0000
+++ doc/changes.rst	2012-06-07 08:20:28 +0000
@@ -1,6 +1,12 @@
 Version History
 ***************
 
+Version 1.8 (Unreleased)
+========================
+
+* Open for development
+* Add 1.4 document format :ref:`format_1_4_schema` see :ref:`difference_between_1_3_and_1_4`
+
 Version 1.7.2
 =============
 
@@ -11,7 +17,6 @@
 * Add a glossary section
 * Add versiontools to install requires so that rtfd.org builds this right
 
-
 Version 1.7
 ===========
 

=== modified file 'doc/index.rst'
--- doc/index.rst	2011-10-21 17:31:23 +0000
+++ doc/index.rst	2012-06-07 08:20:28 +0000
@@ -1,7 +1,7 @@
 Linaro Dashboard Bundle Documentation 
 *************************************
 
-.. seealso:: To see 1.3 format documentation see :ref:`format_1_3_schema`
+.. seealso:: To see 1.4 format documentation see :ref:`format_1_4_schema`
 
 .. toctree::
     :maxdepth: 2

=== modified file 'doc/schema.rst'
--- doc/schema.rst	2011-10-21 01:52:42 +0000
+++ doc/schema.rst	2012-06-07 08:20:28 +0000
@@ -20,5 +20,6 @@
     :maxdepth: 2
 
     schema/docs.rst
+    schema/changes.rst
     schema/raw.rst
     schema/examples.rst

=== added file 'doc/schema/changes.rst'
--- doc/schema/changes.rst	1970-01-01 00:00:00 +0000
+++ doc/schema/changes.rst	2012-06-07 08:20:28 +0000
@@ -0,0 +1,12 @@
+.. _difference_between_1_3_and_1_4:
+
+Differences between 1.3 and 1.4
+===============================
+
+Only one property was added. A test run object can now encode the precise
+duration of the whole test. For example:
+
+.. literalinclude:: ../../linaro_dashboard_bundle/test_documents/example_1.4_test_duration.json
+    :language: javascript
+    :linenos:
+

=== modified file 'doc/schema/raw.rst'
--- doc/schema/raw.rst	2011-10-21 01:52:42 +0000
+++ doc/schema/raw.rst	2012-06-07 08:20:28 +0000
@@ -1,13 +1,17 @@
+.. _format_1_4_schema:
+
+Raw Schema (1.4)
+****************
+
+.. literalinclude:: ../../linaro_dashboard_bundle/schemas/dashboard_bundle_format_1.4.json
+    :language: javascript
+    :linenos:
+
 .. _format_1_3_schema:
 
 Raw Schema (1.3)
 ****************
 
-For those versed in JSON-Schema we'd like to
-recommend reading the full schema directly. Note
-that this is the latest format schema, the full
-catalogue is included in the source distribution.
-
 .. literalinclude:: ../../linaro_dashboard_bundle/schemas/dashboard_bundle_format_1.3.json
     :language: javascript
     :linenos:

=== modified file 'linaro_dashboard_bundle/__init__.py'
--- linaro_dashboard_bundle/__init__.py	2012-04-02 16:39:49 +0000
+++ linaro_dashboard_bundle/__init__.py	2012-06-07 08:20:28 +0000
@@ -25,4 +25,4 @@
 """
 
 
-__version__ = (1, 7, 2, "final", 0)
+__version__ = (1, 8, 0, "dev", 0)

=== modified file 'linaro_dashboard_bundle/evolution.py'
--- linaro_dashboard_bundle/evolution.py	2011-09-19 18:12:59 +0000
+++ linaro_dashboard_bundle/evolution.py	2012-06-07 08:20:28 +0000
@@ -150,6 +150,15 @@
         """
         assert doc.get("format") == "Dashboard Bundle Format 1.2"
         doc["format"] = "Dashboard Bundle Format 1.3"
+    
+    def _evolution_from_1_3_to_1_4(doc):
+        """
+        Evolution method for 1.3 -> 1.4:
+            
+            * No changes required
+        """
+        assert doc.get("format") == "Dashboard Bundle Format 1.3"
+        doc["format"] = "Dashboard Bundle Format 1.4"
 
     EVOLUTION_PATH = [
         ("Dashboard Bundle Format 1.0",
@@ -164,4 +173,7 @@
         ("Dashboard Bundle Format 1.2",
          "Dashboard Bundle Format 1.3",
          _evolution_from_1_2_to_1_3),
+        ("Dashboard Bundle Format 1.3",
+         "Dashboard Bundle Format 1.4",
+         _evolution_from_1_3_to_1_4),
     ]

=== modified file 'linaro_dashboard_bundle/io.py'
--- linaro_dashboard_bundle/io.py	2012-04-02 16:39:49 +0000
+++ linaro_dashboard_bundle/io.py	2012-06-07 08:20:28 +0000
@@ -61,6 +61,11 @@
                 resource_string(
                     __name__,
                     'schemas/dashboard_bundle_format_1.3.json'))),
+        'Dashboard Bundle Format 1.4': Schema(
+            json.loads(
+                resource_string(
+                    __name__,
+                    'schemas/dashboard_bundle_format_1.4.json'))),
     }
 
     @classmethod

=== added file 'linaro_dashboard_bundle/schemas/dashboard_bundle_format_1.4.json'
--- linaro_dashboard_bundle/schemas/dashboard_bundle_format_1.4.json	1970-01-01 00:00:00 +0000
+++ linaro_dashboard_bundle/schemas/dashboard_bundle_format_1.4.json	2012-06-07 08:20:28 +0000
@@ -0,0 +1,268 @@
+{
+    "description": "DashboadBundle object",
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+        "format": {
+            "description": "Document format identifier.",
+            "type": "string",
+            "enum": [
+                "Dashboard Bundle Format 1.4"
+            ]
+        },
+        "test_runs": {
+            "description": "Array of TestRun objects",
+            "type": "array",
+            "optional": true,
+            "items": {
+                "description": "TestRun object",
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                    "analyzer_assigned_uuid": {
+                        "description": "UUID that was assigned by the log analyzer during processing",
+                        "type": "string"
+                    },
+                    "analyzer_assigned_date": {
+                        "description": "Time stamp in ISO 8601 format that was assigned by the log analyzer during processing. The exact format is YYYY-MM-DDThh:mm:ssZ",
+                        "type": "string",
+                        "format": "date-time"
+                    },
+                    "time_check_performed": {
+                        "description": "Indicator on whether the log analyzer had accurate time information",
+                        "type": "boolean"
+                    },
+                    "test_duration": {
+                      "description": "Duration of the test run. Duration is stored in the following format '[DAYS]d [SECONDS]s [MICROSECONDS]us'",
+                      "type": "string",
+                      "optional": true
+                    },
+                    "attributes": {
+                        "description": "Container for additional attributes defined by the test and their values during this particular run",
+                        "type": "object",
+                        "optional": true,
+                        "additionalProperties": {
+                            "description": "Arbitrary properties that are defined by the test",
+                            "type": "string"
+                        }
+                    },
+                    "tags": {
+                        "description": "An optional array of tags that are associated with this test run. Note that those are not version-control tags but arbitrary classifiers for test results",
+                        "type": "array",
+                        "optional": true,
+                        "items": {
+                            "description": "Tag name",
+                            "type": "string"
+                        }
+                    },
+                    "test_id": {
+                        "description": "Test identifier. Must be a well-defined (in scope of the dashboard) name of the test",
+                        "type": "string"
+                    },
+                    "test_results": {
+                        "description": "Array of TestResult objects",
+                        "type": "array",
+                        "items": {
+                            "description": "TestResult object",
+                            "type": "object",
+                            "additionalProperties": false,
+                            "properties": {
+                                "test_case_id": {
+                                    "description": "Identifier of the TestCase this test result came from",
+                                    "type": "string",
+                                    "optional": true
+                                },
+                                "result": {
+                                    "description": "Status code of this test result",
+                                    "type": "string",
+                                    "enum": ["pass", "fail", "skip", "unknown"]
+                                },
+                                "message": {
+                                    "description": "Message scrubbed from the log file",
+                                    "type": "string",
+                                    "optional": true
+                                },
+                                "measurement": {
+                                    "description": "Numerical measurement associated with the test result",
+                                    "type": "number",
+                                    "optional": true,
+                                    "requires": "test_case_id"
+                                },
+                                "units": {
+                                    "description": "Units for measurement",
+                                    "type": "string",
+                                    "optional": true,
+                                    "requires": "measurement"
+                                },
+                                "timestamp": {
+                                    "description": "Date and time when the test was performed",
+                                    "type": "string",
+                                    "optional": true,
+                                    "format": "date-time"
+                                },
+                                "duration": {
+                                    "description": "Duration of the test case. Duration is stored in the following format '[DAYS]d [SECONDS]s [MICROSECONDS]us'",
+                                    "type": "string",
+                                    "optional": true
+                                },
+                                "log_filename": {
+                                    "description": "Filename of the log file which this test result was scrubbed from",
+                                    "type": "string",
+                                    "optional": true
+                                },
+                                "log_lineno": {
+                                    "description": "Precise location in the log file (line number)",
+                                    "type": "integer",
+                                    "optional": true,
+                                    "requires": "log_filename"
+                                },
+                                "attributes": {
+                                    "description": "Container for additional attributes defined by test result",
+                                    "type": "object",
+                                    "optional": true,
+                                    "additionalProperties": {
+                                        "description": "Arbitrary properties that are defined by the particular test result",
+                                        "type": "string"
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    "attachments": {
+                        "description": "Array of attachments",
+                        "optional": true,
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "additionalProperties": false,
+                            "properties": {
+                                "pathname": {
+                                    "description": "Attachment pathname",
+                                    "type": "string"
+                                },
+                                "mime_type": {
+                                     "description": "Attachment MIME type",
+                                     "type": "string"
+                                },
+                                "content": {
+                                    "description": "Attachment content encoded as base64 string with standard encoding",
+                                    "type": "string",
+                                    "optional": true
+                                },
+                                "public_url": {
+                                    "description": "Public URL of this attachment",
+                                    "type": "string",
+                                    "optional": true
+                                }
+                            }
+                        }
+                    },
+                    "hardware_context": {
+                        "description": "Description of the hardware context in which this test was running",
+                        "type": "object",
+                        "optional": true,
+                        "additionalProperties": false,
+                        "properties": {
+                            "devices": {
+                                "description": "Array of HardwareDevice objects",
+                                "type": "array",
+                                "items": {
+                                    "description": "HardwareDevice object",
+                                    "type": "object",
+                                    "properties": {
+                                        "device_type": {
+                                            "type": "string",
+                                            "description": "Device type"
+                                        },
+                                        "description": {
+                                            "type": "string",
+                                            "description": "Human readable description of the device"
+                                        },
+                                        "attributes": {
+                                            "description": "Container for additional attributes defined by the device",
+                                            "type": "object",
+                                            "optional": true,
+                                            "additionalProperties": {
+                                                "description": "Arbitrary properties that are defined by the particular hardware device",
+                                                "type": ["number", "string"]
+                                            }
+                                        }
+                                    },
+                                    "additionalProperties": false
+                                }
+                            }
+                        }
+                    },
+                    "software_context": {
+                        "description": "Description of the software context in which this test was running",
+                        "type": "object",
+                        "additionalProperties": false,
+                        "optional": true,
+                        "properties": {
+                            "image": {
+                                "description": "SoftwareImage object",
+                                "type": "object",
+                                "optional": true,
+                                "additionalProperties": false,
+                                "properties": {
+                                    "name": {
+                                        "description": "Name of the operating system image",
+                                        "type": "string"
+                                    }
+                                }
+                            },
+                            "sources": {
+                                "type": "array",
+                                "optional": true,
+                                "items": {
+                                    "type": "object",
+                                    "additionalProperties": false,
+                                    "properties": {
+                                        "project_name": {
+                                            "type": "string"
+                                        },
+                                        "branch_vcs": {
+                                            "type": "string",
+                                            "enum": ["bzr", "git", "svn"]
+                                        },
+                                        "branch_url": {
+                                            "type": "string"
+                                        },
+                                        "branch_revision": {
+                                            "type": ["string", "integer"]
+                                        },
+                                        "commit_timestamp": {
+                                            "type": "string",
+                                            "format": "date-time",
+                                            "optional": true
+                                        }
+                                    }
+                                }
+                            },
+                            "packages": {
+                                "description": "Array of SoftwarePackage objects",
+                                "type": "array",
+                                "optional": true,
+                                "items": {
+                                    "description": "SoftwarePackage object",
+                                    "type": "object",
+                                    "additionalProperties": false,
+                                    "properties": {
+                                        "name": {
+                                            "description": "Package name",
+                                            "type": "string"
+                                        },
+                                        "version": {
+                                            "description": "Package version",
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

=== added file 'linaro_dashboard_bundle/test_documents/everything_in_one_bundle_1.4.json'
--- linaro_dashboard_bundle/test_documents/everything_in_one_bundle_1.4.json	1970-01-01 00:00:00 +0000
+++ linaro_dashboard_bundle/test_documents/everything_in_one_bundle_1.4.json	2012-06-07 08:20:28 +0000
@@ -0,0 +1,85 @@
+{
+  "format": "Dashboard Bundle Format 1.4", 
+  "test_runs": [
+    {
+      "analyzer_assigned_date": "2010-11-14T13:42:31Z", 
+      "analyzer_assigned_uuid": "01234567-0123-0123-0123-01234567890A", 
+      "test_id": "example test id", 
+      "time_check_performed": false, 
+      "test_duration": "0d 1s 134us", 
+      "attributes": {
+        "attr1": "value1", 
+        "attr2": "value2"
+      }, 
+      "tags": [
+        "example-content", 
+        "example-content-format-1.3"
+      ], 
+      "attachments": [
+        {
+          "pathname": "attachment1.txt", 
+          "mime_type": "text/plain", 
+          "content": "bGluZTEKbGluZTIKbGluZTMK"
+        }, 
+        {
+          "pathname": "attachment2.txt", 
+          "mime_type": "text/plain", 
+          "content": "b3RoZXIgbGluZTEK", 
+          "public_url": "http://www.example.org/attachment2.txt"
+        }
+      ], 
+      "test_results": [
+        {
+          "test_case_id": "example test case id", 
+          "result": "pass", 
+          "message": "example message", 
+          "measurement": 3.5, 
+          "units": "s", 
+          "timestamp": "2010-11-14T13:49:56Z", 
+          "duration": "0d 1s 134us", 
+          "log_filename": "attachment1.txt", 
+          "log_lineno": 2, 
+          "attributes": {
+            "test result attribute": "value"
+          }
+        }
+      ], 
+      "hardware_context": {
+        "devices": [
+          {
+            "device_type": "example device type", 
+            "description": "example device", 
+            "attributes": {
+              "hw attr1": "value1", 
+              "hw attr2": "value2"
+            }
+          }
+        ]
+      }, 
+      "software_context": {
+        "packages": [
+          {
+            "name": "pkg1", 
+            "version": "version1"
+          }, 
+          {
+            "name": "pkg2", 
+            "version": "version2"
+          }
+        ], 
+        "image": {
+          "name": "example os image"
+        }, 
+        "sources": [
+          {
+            "branch_revision": 93556, 
+            "branch_url": "lp:gcc-linaro/4.4", 
+            "branch_vcs": "bzr", 
+            "commit_timestamp": "2010-09-07T14:49:43Z", 
+            "project_name": "linaro-gcc"
+          }
+        ]
+      }
+    }
+  ]
+}

=== added file 'linaro_dashboard_bundle/test_documents/evolution_1.4.json'
--- linaro_dashboard_bundle/test_documents/evolution_1.4.json	1970-01-01 00:00:00 +0000
+++ linaro_dashboard_bundle/test_documents/evolution_1.4.json	2012-06-07 08:20:28 +0000
@@ -0,0 +1,70 @@
+{
+  "format": "Dashboard Bundle Format 1.4", 
+  "test_runs": [
+    {
+      "analyzer_assigned_date": "2010-11-14T13:42:31Z", 
+      "analyzer_assigned_uuid": "01234567-0123-0123-0123-01234567890A", 
+      "test_id": "example test id", 
+      "time_check_performed": false, 
+      "attributes": {
+        "attr1": "value1", 
+        "attr2": "value2"
+      }, 
+      "attachments": [
+        {
+          "pathname": "attachment1.txt", 
+          "mime_type": "text/plain", 
+          "content": "bGluZTEKbGluZTIKbGluZTMK"
+        }, 
+        {
+          "pathname": "attachment2.txt", 
+          "mime_type": "text/plain", 
+          "content": "b3RoZXIgbGluZTEK"
+        }
+      ], 
+      "test_results": [
+        {
+          "test_case_id": "example test case id", 
+          "result": "pass", 
+          "message": "example message", 
+          "measurement": 3.5, 
+          "units": "s", 
+          "timestamp": "2010-11-14T13:49:56Z", 
+          "duration": "0d 1s 134us", 
+          "log_filename": "attachment1.txt", 
+          "log_lineno": 2, 
+          "attributes": {
+            "test result attribute": "value"
+          }
+        }
+      ], 
+      "hardware_context": {
+        "devices": [
+          {
+            "device_type": "example device type", 
+            "description": "example device", 
+            "attributes": {
+              "hw attr1": "value1", 
+              "hw attr2": "value2"
+            }
+          }
+        ]
+      }, 
+      "software_context": {
+        "packages": [
+          {
+            "name": "pkg1", 
+            "version": "version1"
+          }, 
+          {
+            "name": "pkg2", 
+            "version": "version2"
+          }
+        ], 
+        "image": {
+          "name": "example os image"
+        }
+      }
+    }
+  ]
+}

=== added file 'linaro_dashboard_bundle/test_documents/example_1.4_test_duration.json'
--- linaro_dashboard_bundle/test_documents/example_1.4_test_duration.json	1970-01-01 00:00:00 +0000
+++ linaro_dashboard_bundle/test_documents/example_1.4_test_duration.json	2012-06-07 08:20:28 +0000
@@ -0,0 +1,8 @@
+{
+  "format": "Dashboard Bundle Format 1.4", 
+  "test_runs": [
+    {
+      "test_duration": "0d 1s 134us", 
+    }
+  ]
+}

=== modified file 'linaro_dashboard_bundle/tests.py'
--- linaro_dashboard_bundle/tests.py	2012-04-02 16:39:49 +0000
+++ linaro_dashboard_bundle/tests.py	2012-06-07 08:20:28 +0000
@@ -386,17 +386,50 @@
     def test_evolved_document_is_latest_format(self):
         self.assertFalse(DocumentEvolution.is_latest(self.doc))
         DocumentEvolution.evolve_document(self.doc, one_step=True)
+        self.assertFalse(DocumentEvolution.is_latest(self.doc))
+
+    def test_evolved_document_is_valid(self):
+        DocumentEvolution.evolve_document(self.doc, one_step=True)
+        self.assertEqual(DocumentIO.check(self.doc),
+                         "Dashboard Bundle Format 1.3")
+
+    def test_evoloved_document_is_what_we_expect(self):
+        DocumentEvolution.evolve_document(self.doc, one_step=True)
+        fmt, evolved_doc = DocumentIO.load(
+            resource_stream('linaro_dashboard_bundle',
+                            'test_documents/evolution_1.3.json'),
+            retain_order=False)
+        self.assertEqual(self.doc, evolved_doc)
+
+
+class DocumentEvolutionTests_1_3_to_1_4(TestCase):
+
+    def setUp(self):
+        super(DocumentEvolutionTests_1_3_to_1_4, self).setUp()
+        self.fmt, self.doc = DocumentIO.load(
+            resource_stream('linaro_dashboard_bundle',
+                            'test_documents/evolution_1.3.json'),
+            retain_order=False)
+
+    def test_format_is_changed(self):
+        self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.3")
+        DocumentEvolution.evolve_document(self.doc, one_step=True)
+        self.assertEqual(self.doc["format"], "Dashboard Bundle Format 1.4")
+
+    def test_evolved_document_is_latest_format(self):
+        self.assertFalse(DocumentEvolution.is_latest(self.doc))
+        DocumentEvolution.evolve_document(self.doc, one_step=True)
         self.assertTrue(DocumentEvolution.is_latest(self.doc))
 
     def test_evolved_document_is_valid(self):
         DocumentEvolution.evolve_document(self.doc, one_step=True)
         self.assertEqual(DocumentIO.check(self.doc),
-                         "Dashboard Bundle Format 1.3")
+                         "Dashboard Bundle Format 1.4")
 
     def test_evoloved_document_is_what_we_expect(self):
         DocumentEvolution.evolve_document(self.doc, one_step=True)
         fmt, evolved_doc = DocumentIO.load(
             resource_stream('linaro_dashboard_bundle',
-                            'test_documents/evolution_1.3.json'),
+                            'test_documents/evolution_1.4.json'),
             retain_order=False)
         self.assertEqual(self.doc, evolved_doc)