1
=== modified file 'plainbox/plainbox/impl/integration_tests.py'
2
--- plainbox/plainbox/impl/integration_tests.py	2013-04-04 17:12:13 +0000
3
+++ plainbox/plainbox/impl/integration_tests.py	2013-07-05 11:51:23 +0000
4
@@ -38,11 +38,21 @@
5
38
from plainbox.testing_utils.cwd import TestCwd
38
from plainbox.testing_utils.cwd import TestCwd
6
39
from plainbox.testing_utils.io import TestIO
39
from plainbox.testing_utils.io import TestIO
7
40
from plainbox.testing_utils.testcases import TestCaseWithParameters
40
from plainbox.testing_utils.testcases import TestCaseWithParameters
8
41
from plainbox.testing_utils.resource import ResourceCache
9
41
42
10
42
43
11
43
class IntegrationTests(TestCaseWithParameters):
44
class IntegrationTests(TestCaseWithParameters):
14
44
45
    """
15
45
    parameter_names = ('job_name',)
46
    Test cases for checking execution and outcome of checkbox jobs.
16
47
    Each test case is parametrized by the job name and execution "profile".
17
48
18
49
    The profile is simply a string that somehow characterizes where this test
19
50
    is applicable.
20
51
    """
21
52
22
53
    cache = ResourceCache()
23
54
24
55
    parameter_names = ('scenario_pathname',)
25
46
56
26
47
    def setUp(self):
57
    def setUp(self):
27
48
        # session data are kept in XDG_CACHE_HOME/plainbox/.session
58
        # session data are kept in XDG_CACHE_HOME/plainbox/.session
28
@@ -51,24 +61,58 @@
29
51
        self._sandbox = tempfile.mkdtemp()
61
        self._sandbox = tempfile.mkdtemp()
30
52
        self._env = os.environ
62
        self._env = os.environ
31
53
        os.environ['XDG_CACHE_HOME'] = self._sandbox
63
        os.environ['XDG_CACHE_HOME'] = self._sandbox
32
64
        # Load the expected results and keep them in memory
33
65
        self.scenario_data = self.cache.get(
34
66
            key=('scenario_data', self.parameters.scenario_pathname),
35
67
            operation=lambda: load_scenario_data(
36
68
                self.parameters.scenario_pathname))
37
69
        # Skip tests that are not applicable for the current system
38
70
        self.skip_if_incompatible()
39
71
        # Execute the job and remember the results.
40
72
        (self.job_result, self.job_return_code, self.job_stdout,
41
73
         self.job_stderr) = self.cache.get(
42
74
             key=('job-run-artifacts', self.parameters.scenario_pathname),
43
75
             operation=lambda: execute_job(self.scenario_data['job_name']))
44
76
45
77
    def test_job_result(self):
46
78
        # Check that results match expected values
47
79
        self.assertEqual(self.job_result, self.scenario_data.get("result"))
48
80
49
81
    def test_job_return_code(self):
50
82
        # Check the return code for correctness
51
83
        self.assertEqual(self.job_return_code,
52
84
                         self.scenario_data.get("return_code", 0))
53
85
54
86
    def skip_if_incompatible(self):
55
87
        """
56
88
        Skip a job if it is incompatible with the current environment
57
89
        """
58
90
        if self.scenario_data.get('profile') != 'default':
59
91
            self.skipTest("not applicable for current profile")
60
54
92
61
55
    @classmethod
93
    @classmethod
64
56
    def _gen_job_name_values(cls, package='plainbox',
94
    def _discover_test_scenarios(cls, package='plainbox',
65
57
                             root='test-data/integration-tests/'):
95
                                 dirname="/test-data/integration-tests/",
66
96
                                 extension=".json"):
67
58
        """
97
        """
69
59
        Discover job names for jobs that we have reference data for
98
        Discover test scenarios.
70
99
71
100
        Generates special absolute pathnames to scenario files. All those paths
72
101
        are really relative to the plainbox package. Those pathnames are
73
102
        suitable for pkg_resources.resource_ functions.
74
60
103
75
61
        All reference data should be dropped to
104
        All reference data should be dropped to
76
62
        ``plainbox/test-data/integration-tests/`` as a json file
105
        ``plainbox/test-data/integration-tests/`` as a json file
77
63
        """
106
        """
82
64
        for name in resource_listdir(package, root):
107
        for name in resource_listdir(package, dirname):
83
65
            resource_name = os.path.join(root, name)
108
            resource_pathname = os.path.join(dirname, name)
84
66
            if resource_isdir(package, resource_name):
109
            if resource_isdir(package, resource_pathname):
85
67
                for item in cls._gen_job_name_values(package, resource_name):
110
                for item in cls._discover_test_scenarios(package,
86
111
                                                         resource_pathname,
87
112
                                                         extension):
88
68
                    yield item
113
                    yield item
92
69
            elif resource_name.endswith('.json'):
114
            elif resource_pathname.endswith(extension):
93
70
                yield resource_name[
115
                yield resource_pathname
91
71
                    len('test-data/integration-tests/'):-len('.json')]
94
72
116
95
73
    @classmethod
117
    @classmethod
96
74
    def get_parameter_values(cls):
118
    def get_parameter_values(cls):
97
@@ -77,39 +121,57 @@
98
77
121
99
78
        Creates subsequent tuples for each job that has reference data
122
        Creates subsequent tuples for each job that has reference data
100
79
        """
123
        """
133
80
        for job_name in cls._gen_job_name_values():
124
        for scenario_pathname in cls._discover_test_scenarios():
134
81
            yield (job_name,)
125
            yield (scenario_pathname,)
103
82
104
83
    def test_job_result(self):
105
84
        # Create a scratch directory so that we can save results there. The
106
85
        # shared directory is also used for running tests as some test jobs
107
86
        # leave junk around the current directory.
108
87
        with TemporaryDirectory() as scratch_dir:
109
88
            # Save results to results.json in the scratch directory
110
89
            actual_results_path = os.path.join(scratch_dir, 'results.json')
111
90
            # Redirect all standard IO so that the test is silent.
112
91
            # Run the script, having relocated to the scratch directory
113
92
            # Capture SystemExit that is always raised by main() so that
114
93
            # we can observe the return code as well.
115
94
            with TestIO(), TestCwd(scratch_dir),\
116
95
                    self.assertRaises(SystemExit) as call:
117
96
                main(['run', '-i', self.parameters.job_name,
118
97
                      '--output-format=json', '-o', actual_results_path])
119
98
            # Check the return code for correctness
120
99
            self.assertEqual(call.exception.args, (0,))
121
100
            # Load the actual results and keep them in memory
122
101
            with open(actual_results_path, encoding='UTF-8') as stream:
123
102
                actual_result = json.load(stream)
124
103
        # [ At this time TestIO and TemporaryDirectory are gone ]
125
104
        # Load the expected results and keep them in memory
126
105
        reference_path = resource_filename(
127
106
            "plainbox", "test-data/integration-tests/{}.json".format(
128
107
                self.parameters.job_name))
129
108
        with open(reference_path, encoding='UTF-8') as stream:
130
109
            expected_result = json.load(stream)
131
110
        # Check that results match expected values
132
111
        self.assertEqual(actual_result, expected_result)
135
112
126
136
113
    def tearDown(self):
127
    def tearDown(self):
137
114
        shutil.rmtree(self._sandbox)
128
        shutil.rmtree(self._sandbox)
138
115
        os.environ = self._env
129
        os.environ = self._env
139
130
140
131
141
132
def load_scenario_data(scenario_pathname):
142
133
    """
143
134
    Load and return scenario data.
144
135
145
136
    Data is loaded from a .json file located in the plainbox package
146
137
    directory. Individual files are named after the jobs they describe.
147
138
    """
148
139
    pathname = resource_filename("plainbox", scenario_pathname)
149
140
    with open(pathname, encoding='UTF-8') as stream:
150
141
        return json.load(stream)
151
142
152
143
153
144
def execute_job(job_name):
154
145
    """
155
146
    Execute the specified job.
156
147
157
148
    The job is invoked using a high-level interface from box so the test will
158
149
    actually execute the same way as the UI would execute it. It will
159
150
    create/tear-down appropriate session objects as well.
160
151
161
152
    Returns (result, return_code) where result is the deserialized JSON saved
162
153
    at the end of the job.
163
154
    """
164
155
    # Create a scratch directory so that we can save results there. The
165
156
    # shared directory is also used for running tests as some test jobs
166
157
    # leave junk around the current directory.
167
158
    with TemporaryDirectory() as scratch_dir:
168
159
        # Save results to results.json in the scratch directory
169
160
        pathname = os.path.join(scratch_dir, 'results.json')
170
161
        # Redirect all standard IO so that the test is silent.
171
162
        # Run the script, having relocated to the scratch directory
172
163
        with TestIO() as io, TestCwd(scratch_dir):
173
164
            try:
174
165
                main(['run', '-i', job_name,
175
166
                      '--output-format=json', '-o', pathname])
176
167
            except SystemExit as exc:
177
168
                # Capture SystemExit that is always raised by main() so that we
178
169
                # can observe the return code as well.
179
170
                job_return_code = exc.args[0]
180
171
            else:
181
172
                job_return_code = None
182
173
        # Load the actual results and keep them in memory
183
174
        with open(pathname, encoding='UTF-8') as stream:
184
175
            job_result = json.load(stream)
185
176
    # [ At this time TestIO and TemporaryDirectory are gone ]
186
177
    return job_result, job_return_code, io.stdout, io.stderr
187
116
178
188
=== modified file 'plainbox/plainbox/test-data/integration-tests/cpu/scaling_test.json'
189
--- plainbox/plainbox/test-data/integration-tests/cpu/scaling_test.json	2013-03-15 15:49:18 +0000
190
+++ plainbox/plainbox/test-data/integration-tests/cpu/scaling_test.json	2013-07-05 11:51:23 +0000
191
@@ -1,10 +1,14 @@
192
1
{
1
{
199
2
    "result_map": {
2
    "profile": "default",
200
3
        "cpu/scaling_test": {
3
    "job_name": "cpu/scaling_test",
201
4
            "outcome": "pass"
4
    "result": {
202
5
        }, 
5
        "result_map": {
203
6
        "package": {
6
            "cpu/scaling_test": {
204
7
            "outcome": "pass"
7
                "outcome": "pass"
205
8
            }, 
206
9
            "package": {
207
10
                "outcome": "pass"
208
11
            }
209
8
        }
12
        }
210
9
    }
13
    }
211
10
}
212
11
\ No newline at end of file
14
\ No newline at end of file
213
15
}
214
12
16
215
=== added file 'plainbox/plainbox/testing_utils/resource.py'
216
--- plainbox/plainbox/testing_utils/resource.py	1970-01-01 00:00:00 +0000
217
+++ plainbox/plainbox/testing_utils/resource.py	2013-07-05 11:51:23 +0000
218
@@ -0,0 +1,60 @@
219
1
# This file is part of Checkbox.
220
2
#
221
3
# Copyright 2013 Canonical Ltd.
222
4
# Written by:
223
5
#   Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
224
6
#
225
7
# Checkbox is free software: you can redistribute it and/or modify
226
8
# it under the terms of the GNU General Public License as published by
227
9
# the Free Software Foundation, either version 3 of the License, or
228
10
# (at your option) any later version.
229
11
#
230
12
# Checkbox is distributed in the hope that it will be useful,
231
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
232
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
233
15
# GNU General Public License for more details.
234
16
#
235
17
# You should have received a copy of the GNU General Public License
236
18
# along with Checkbox.  If not, see <http://www.gnu.org/licenses/>.
237
19
238
20
"""
239
21
plainbox.testing_utils.resource
240
22
===============================
241
23
242
24
Implementation of simple resource sharing cache for unit tests
243
25
"""
244
26
245
27
import weakref
246
28
247
29
248
30
class ResourceCache:
249
31
    """
250
32
    Mix-in for having cached resources that are shared and reused by all
251
33
    instances. If your test needs to compute something (slowly) and reuse
252
34
    it in various different test_ methods then this will save time.
253
35
    """
254
36
255
37
    def __init__(self):
256
38
        """
257
39
        Initialize a new ResourceCache object
258
40
        """
259
41
        # XXX: it would be nice to have something like true cache semantics of
260
42
        # java's SoftReference system. We do the second best thing which is to
261
43
        # use weak references on the values held in the cache.
262
44
        # self.weak_cache = weakref.WeakValueDictionary()
263
45
        self.weak_cache = {}
264
46
265
47
    def get(self, key, operation):
266
48
        """
267
49
        Get a value from the cache, falling back to computing it if needed
268
50
269
51
        Gets something from the cache dictionary, referenced by the key. If the
270
52
        value is missing it is computed, by calling the operation, and stored
271
53
        in the cache.
272
54
        """
273
55
        try:
274
56
            value = self.weak_cache[key]
275
57
        except KeyError:
276
58
            value = operation()
277
59
            self.weak_cache[key] = value
278
60
        return value
Status:	Work in progress
Proposed branch:	lp:~zyga/checkbox/device-profiles
Merge into:	lp:checkbox
Diff against target:	278 lines (+177/-51) 3 files modified plainbox/plainbox/impl/integration_tests.py (+106/-44) plainbox/plainbox/test-data/integration-tests/cpu/scaling_test.json (+11/-7) plainbox/plainbox/testing_utils/resource.py (+60/-0)
To merge this branch:	bzr merge lp:~zyga/checkbox/device-profiles
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
Checkbox Developers		2013-02-18	Pending
Review via email: mp+149016@code.launchpad.net