Merge lp:~barry/ubuntu-system-image/citrain30 into lp:~ubuntu-managed-branches/ubuntu-system-image/system-image

Proposed by Barry Warsaw
Status: Merged
Approved by: Barry Warsaw
Approved revision: 252
Merged at revision: 241
Proposed branch: lp:~barry/ubuntu-system-image/citrain30
Merge into: lp:~ubuntu-managed-branches/ubuntu-system-image/system-image
Diff against target: 15774 lines (+8086/-2651)
138 files modified
MANIFEST.in (+1/-1)
NEWS.rst (+72/-1)
PKG-INFO (+1/-1)
cli-manpage.rst (+31/-18)
coverage-curl.ini (+21/-0)
coverage-udm.ini (+21/-0)
coverage.ini (+0/-15)
dbus-manpage.rst (+50/-28)
debian/changelog (+77/-0)
debian/control (+3/-2)
debian/rules (+15/-10)
debian/tests/control (+7/-7)
debian/tests/dryrun (+1/-1)
debian/tests/prep.py (+9/-2)
debian/tests/smoketest (+2/-2)
debian/tests/smoketest-noreboot (+2/-2)
debian/tests/unittests (+1/-1)
ini-manpage.rst (+33/-30)
setup.cfg (+1/-1)
setup.py (+1/-1)
system_image.egg-info/PKG-INFO (+1/-1)
system_image.egg-info/SOURCES.txt (+105/-59)
systemimage/api.py (+20/-10)
systemimage/apply.py (+32/-9)
systemimage/bag.py (+7/-1)
systemimage/candidates.py (+1/-1)
systemimage/channel.py (+1/-1)
systemimage/config.py (+174/-108)
systemimage/curl.py (+275/-0)
systemimage/data/client.ini (+0/-35)
systemimage/dbus.py (+100/-95)
systemimage/device.py (+1/-1)
systemimage/docs/conf.py (+1/-1)
systemimage/download.py (+119/-208)
systemimage/gpg.py (+6/-1)
systemimage/helpers.py (+53/-43)
systemimage/image.py (+5/-1)
systemimage/index.py (+2/-5)
systemimage/keyring.py (+5/-5)
systemimage/logging.py (+4/-2)
systemimage/main.py (+117/-47)
systemimage/reactor.py (+7/-4)
systemimage/scores.py (+32/-15)
systemimage/service.py (+15/-18)
systemimage/settings.py (+1/-1)
systemimage/state.py (+32/-23)
systemimage/testing/controller.py (+95/-45)
systemimage/testing/dbus.py (+22/-10)
systemimage/testing/demo.py (+6/-6)
systemimage/testing/helpers.py (+174/-62)
systemimage/testing/nose.py (+7/-2)
systemimage/testing/service.py (+0/-50)
systemimage/tests/data/00.ini (+1/-2)
systemimage/tests/data/01.ini (+2/-3)
systemimage/tests/data/api.channels_01.json (+13/-0)
systemimage/tests/data/api.index_01.json (+36/-0)
systemimage/tests/data/api.index_02.json (+251/-0)
systemimage/tests/data/api.index_03.json (+37/-0)
systemimage/tests/data/candidates.index_01.json (+6/-0)
systemimage/tests/data/candidates.index_02.json (+23/-0)
systemimage/tests/data/candidates.index_08.json (+244/-0)
systemimage/tests/data/candidates.index_10.json (+36/-0)
systemimage/tests/data/candidates.index_11.json (+37/-0)
systemimage/tests/data/candidates.index_13.json (+244/-0)
systemimage/tests/data/channel.channels_01.json (+23/-0)
systemimage/tests/data/channel.channels_02.json (+38/-0)
systemimage/tests/data/channel.channels_03.json (+70/-0)
systemimage/tests/data/channel.channels_04.json (+56/-0)
systemimage/tests/data/channel.channels_05.json (+23/-0)
systemimage/tests/data/channel_06.ini (+0/-8)
systemimage/tests/data/channel_07.ini (+0/-8)
systemimage/tests/data/config.config_01.ini (+34/-0)
systemimage/tests/data/config.config_02.ini (+34/-0)
systemimage/tests/data/config.config_03.ini (+3/-4)
systemimage/tests/data/config.config_04.ini (+36/-0)
systemimage/tests/data/config.config_05.ini (+3/-4)
systemimage/tests/data/config.config_06.ini (+3/-4)
systemimage/tests/data/config.config_07.ini (+1/-2)
systemimage/tests/data/config.config_08.ini (+1/-2)
systemimage/tests/data/config.config_09.ini (+0/-3)
systemimage/tests/data/config.config_10.ini (+27/-0)
systemimage/tests/data/config.config_11.ini (+2/-0)
systemimage/tests/data/config_04.ini (+0/-36)
systemimage/tests/data/config_09.ini (+0/-27)
systemimage/tests/data/config_10.ini (+0/-35)
systemimage/tests/data/dbus.index_03.json (+36/-0)
systemimage/tests/data/dbus.index_06.json (+37/-0)
systemimage/tests/data/download.index_01.json (+6/-0)
systemimage/tests/data/gpg.channels_01.json (+23/-0)
systemimage/tests/data/helpers.config_01.ini (+7/-0)
systemimage/tests/data/helpers.config_02.ini (+6/-0)
systemimage/tests/data/index.channels_01.json (+9/-0)
systemimage/tests/data/index.channels_02.json (+13/-0)
systemimage/tests/data/index.channels_05.json (+9/-0)
systemimage/tests/data/index.index_01.json (+251/-0)
systemimage/tests/data/index.index_04.json (+244/-0)
systemimage/tests/data/main.channels_01.json (+13/-0)
systemimage/tests/data/main.channels_03.json (+13/-0)
systemimage/tests/data/main.config_01.ini (+5/-6)
systemimage/tests/data/main.config_05.ini (+7/-0)
systemimage/tests/data/main.config_07.ini (+1/-1)
systemimage/tests/data/main.index_04.json (+36/-0)
systemimage/tests/data/main.index_05.json (+36/-0)
systemimage/tests/data/scores.index_01.json (+245/-0)
systemimage/tests/data/scores.index_05.json (+245/-0)
systemimage/tests/data/scores.index_06.json (+253/-0)
systemimage/tests/data/scores.index_07.json (+252/-0)
systemimage/tests/data/state.channels_01.json (+64/-0)
systemimage/tests/data/state.channels_02.json (+13/-0)
systemimage/tests/data/state.config_01.ini (+2/-0)
systemimage/tests/data/state.config_02.ini (+6/-0)
systemimage/tests/data/state.index_01.json (+244/-0)
systemimage/tests/data/state.index_02.json (+245/-0)
systemimage/tests/data/state.index_03.json (+36/-0)
systemimage/tests/data/state.index_04.json (+37/-0)
systemimage/tests/data/state.index_07.json (+2/-3)
systemimage/tests/test_api.py (+71/-93)
systemimage/tests/test_bag.py (+1/-1)
systemimage/tests/test_candidates.py (+37/-45)
systemimage/tests/test_channel.py (+19/-19)
systemimage/tests/test_config.py (+202/-178)
systemimage/tests/test_dbus.py (+365/-167)
systemimage/tests/test_download.py (+229/-115)
systemimage/tests/test_gpg.py (+129/-19)
systemimage/tests/test_helpers.py (+183/-171)
systemimage/tests/test_image.py (+1/-1)
systemimage/tests/test_index.py (+32/-68)
systemimage/tests/test_keyring.py (+1/-1)
systemimage/tests/test_main.py (+725/-432)
systemimage/tests/test_scores.py (+127/-15)
systemimage/tests/test_settings.py (+4/-6)
systemimage/tests/test_state.py (+244/-142)
systemimage/tests/test_winner.py (+23/-19)
systemimage/udm.py (+212/-0)
systemimage/version.txt (+1/-1)
tools/demo.ini (+0/-1)
tools/runme.sh (+10/-0)
tox.ini (+21/-16)
To merge this branch: bzr merge lp:~barry/ubuntu-system-image/citrain30
Reviewer Review Type Date Requested Status
Ubuntu CI managed package branches Pending
Review via email: mp+259635@code.launchpad.net

Commit message

system-image 3.0

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'MANIFEST.in'
--- MANIFEST.in 2014-01-30 16:56:57 +0000
+++ MANIFEST.in 2015-05-20 14:55:53 +0000
@@ -1,5 +1,5 @@
1include *.py MANIFEST.in1include *.py MANIFEST.in
2global-include *.txt *.rst *.json *.ini *.gpg *.pem *.service *.in *.conf *.cfg2global-include *.txt *.rst *.json *.ini *.gpg *.pem *.service *.in *.conf *.cfg *.sh
3prune build3prune build
4prune dist4prune dist
5prune .tox5prune .tox
66
=== modified file 'NEWS.rst'
--- NEWS.rst 2014-09-26 14:36:34 +0000
+++ NEWS.rst 2015-05-20 14:55:53 +0000
@@ -2,7 +2,78 @@
2NEWS for system-image updater2NEWS for system-image updater
3=============================3=============================
44
52.5 (2014-XX-XX)53.0 (2015-05-08)
6================
7 * Support a built-in PyCURL-based downloader in addition to the traditional
8 ubuntu-download-manager (over D-BUS) downloader. Auto-detects which
9 downloader to use based on whether udm is available on the system bus,
10 pycurl is importable, and the setting of the SYSTEMIMAGE_PYCURL environment
11 variable. Initial contribution by Michael Vogt. (LP: #1374459)
12 * Support alternative machine-id files as fall backs if the D-Bus file does
13 not exist. Specifically, add systemd's /etc/machine-id to the list.
14 Initial contribution by Michael Vogt. (LP: #1384859)
15 * Support multiple configuration files, as in a `config.d` directory. Now,
16 configuration files are named `NN_whatever.ini` where "NN" must be a
17 numeric prefix. Files are loaded in sorted numeric order, with later files
18 overriding newer files. Support for both the `client.ini` and
19 `channel.ini` files has been removed. (LP: #1373467)
20 * The `[system]build_file` variable has been removed. Build number
21 information now must come from the `.ini` files, and last update date
22 comes from the newest `.ini` file loaded.
23 * The `-C` command line option now takes a path to the configuration
24 directory.
25 * Reworked the checking and downloading locks/flags to so that they will work
26 better with configuration reloading. (LP: #1412698)
27 * Support for the `/etc/ubuntu-build` file has been removed. The build
28 number now comes from the configuration files. (LP: #1377312)
29 * Move the `archive-master.tar.xz` file to `/usr/share/system-image` for
30 better FHS compliance. (LP: #1377184)
31 * Since devices do not always reboot to apply changes, the `[hooks]update`
32 variable has been renamed to `[hooks]apply`. (LP: #1381538)
33 * For testing purposes only, `system-image-cli` now supports an
34 undocumented command line switch `--skip-gpg-verification`. Originally
35 given by Jani Monoses. (LP: #1333414)
36 * A new D-Bus signal `Applied(bool)` is added, which is returned in
37 response to the `ApplyUpdate()` asynchronous method call. For devices
38 which do not need to reboot in order to apply the update, this is the only
39 signal you will get. If your device needs to reboot you will also receive
40 the `Rebooting(bool)` command as with earlier versions. The semantics of
41 the flag argument are the same in both cases, as are the race timing issues
42 inherent in these signals. See the `system-image-dbus(8)` manpage for
43 details. (LP: #1417176)
44 * As part of LP: #1417176, the `--no-reboot` switch for
45 `system-image-cli(1)` has been deprecated. Use `--no-apply` instead
46 (`-g` is still the shortcut).
47 * Support production factory resets. `system-image-cli --production-reset`
48 and a new D-Bus API method `ProductionReset()` are added. Given by Ricardo
49 Salveti. (LP: #1419027)
50 * A new key, `target_version_detail` has been added to the dictionary
51 returned by the `.Information()` D-Bus method. (LP: #1399687)
52 * The `User-Agent` HTTP header now also includes device and channel names.
53 (LP: #1387719)
54 * Added `--progress` flag to `system-image-cli` for specifying methods for
55 reporting progress. Current available values are: `dots` (compatible with
56 system-image 2.5), `logfile` (compatible with system-image 2.5's
57 `--verbose` flag), and `json` for JSON records on stdout. (LP: #1423622)
58 * Support for the `SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS` environment
59 variable has been removed.
60 * Fix `system-image-cli --list-channels`. (LP: #1448153)
61
622.5.1 (2014-10-21)
63==================
64 * Make phased upgrade percentage calculation idempotent for each tuple of
65 (channel, target-build-number, machine-id). Also, modify the candidate
66 upgrade path selection process such that if the lowest scored candidate
67 path has a phased percentage greater than the device's percentage, the
68 candidate will be ignored, and the next lowest scored candidate will be
69 checked until either a winner is found or no candidates are left, in which
70 case the device is deemed to be up-to-date. (LP: #1383539)
71 * `system-image-cli -p/--percentage` is added to allow command line override
72 of the device's phased percentage.
73 * `system-image-cli --dry-run` now also displays the phase percentage of the
74 winning candidate upgrade path.
75
762.5 (2014-09-29)
6================77================
7 * Remove the previously deprecated `system-image-cli --dbus` command line78 * Remove the previously deprecated `system-image-cli --dbus` command line
8 switch. (LP: #1369717)79 switch. (LP: #1369717)
980
=== modified file 'PKG-INFO'
--- PKG-INFO 2014-09-26 14:36:34 +0000
+++ PKG-INFO 2015-05-20 14:55:53 +0000
@@ -1,6 +1,6 @@
1Metadata-Version: 1.01Metadata-Version: 1.0
2Name: system-image2Name: system-image
3Version: 2.53Version: 3.0
4Summary: Ubuntu System Image Based Upgrades4Summary: Ubuntu System Image Based Upgrades
5Home-page: UNKNOWN5Home-page: UNKNOWN
6Author: Barry Warsaw6Author: Barry Warsaw
77
=== modified file 'cli-manpage.rst'
--- cli-manpage.rst 2014-09-17 13:41:31 +0000
+++ cli-manpage.rst 2015-05-20 14:55:53 +0000
@@ -7,9 +7,9 @@
7------------------------------------------------7------------------------------------------------
88
9:Author: Barry Warsaw <barry@ubuntu.com>9:Author: Barry Warsaw <barry@ubuntu.com>
10:Date: 2014-09-1610:Date: 2015-01-15
11:Copyright: 2013-2014 Canonical Ltd.11:Copyright: 2013-2015 Canonical Ltd.
12:Version: 2.412:Version: 3.0
13:Manual section: 113:Manual section: 1
1414
1515
@@ -68,10 +68,17 @@
6868
69-n, --dry-run69-n, --dry-run
70 Calculate and print the upgrade path, but do not download or apply it.70 Calculate and print the upgrade path, but do not download or apply it.
7171 *New in system-image 2.5.1: output displays the target phase percentage*
72--no-reboot72
73 Downloads all files and prepares for a reboot into recovery, but doesn't73-p VALUE, --percentage VALUE
74 actually issue the reboot.74 For testing purposes, force a device specific phase percentage. The value
75 must be an integer between 0 and 100. *New in system-image 2.5.1*
76
77-g, --no-apply
78 Downloads all files and prepares for, but does not actually apply the
79 update. On devices which require a reboot to apply the update, no reboot
80 is performed. *New in system-image 3.0: --no-reboot is renamed to
81 --no-apply*
7582
76-v, --verbose83-v, --verbose
77 Increase the logging verbosity. With one ``-v``, logging goes to the84 Increase the logging verbosity. With one ``-v``, logging goes to the
@@ -79,10 +86,13 @@
79 enabled. With two ``-v`` (or ``-vv``), logging both to the console and to86 enabled. With two ``-v`` (or ``-vv``), logging both to the console and to
80 the log file are output at ``DEBUG`` level.87 the log file are output at ``DEBUG`` level.
8188
82-C FILE, --config FILE89-C DIR, --config DIR
83 Use the given configuration file, otherwise use the default. The program90 Use the given configuration directory, otherwise use the system default.
84 will optionally also read a ``channel.ini`` file in the same directory as91 The program will read all the files in this directory that begin with a
85 ``FILE``.92 number, followed by an underscore, and ending in ``.ini``
93 (e.g. ``03_myconfig.ini``). The files are read in sorted numerical order
94 from lowest prefix number to highest, with later configuration files able
95 to override any variable in any section.
8696
87--factory-reset97--factory-reset
88 Wipes the data partition and issues a reboot into recovery. This98 Wipes the data partition and issues a reboot into recovery. This
@@ -91,6 +101,13 @@
91--show-settings101--show-settings
92 Show all the key/value pairs in the settings database.102 Show all the key/value pairs in the settings database.
93103
104--progress [dots|logfile|json]
105 Report progress in various ways. `dots` prints some dots every once in a
106 while to stderr; this mimic what was available in system-image 2.5.
107 `logfile` prints messages at debug level to the system-image log file, and
108 is also available in 2.5 (via the `--verbose` flag). `json` prints JSON
109 records to stdout. *New in system-image 3.0*
110
94--get KEY111--get KEY
95 Print the value for the given key in the settings database. If the key is112 Print the value for the given key in the settings database. If the key is
96 missing, a default value is printed. May be given multiple times.113 missing, a default value is printed. May be given multiple times.
@@ -107,15 +124,11 @@
107FILES124FILES
108=====125=====
109126
110/etc/system-image/client.ini127/etc/system-image/[0-9]+*.ini
111 Default configuration file.128 Default configuration files.
112
113/etc/system-image/channel.ini
114 Optional configuration file overrides (for the ``[service]`` section
115 only).
116129
117130
118SEE ALSO131SEE ALSO
119========132========
120133
121client.ini(5), system-image-dbus(8)134system-image.ini(5), system-image-dbus(8)
122135
=== added file 'coverage-curl.ini'
--- coverage-curl.ini 1970-01-01 00:00:00 +0000
+++ coverage-curl.ini 2015-05-20 14:55:53 +0000
@@ -0,0 +1,21 @@
1[run]
2branch = true
3parallel = true
4omit =
5 setup*
6 systemimage/data/*
7 systemimage/docs/*
8 systemimage/testing/*
9 systemimage/tests/*
10 systemimage/udm.py
11 /usr/lib/*
12 .tox/coverage-curl/lib/python3.4/distutils/*
13 .tox/coverage-curl/lib/python3.4/site-packages/pkg_resources*
14 .tox/coverage-udm/lib/python3.4/distutils/*
15 .tox/coverage-udm/lib/python3.4/site-packages/pkg_resources*
16
17[paths]
18source =
19 systemimage
20 .tox/coverage-curl/lib/python*/site-packages/systemimage
21 .tox/coverage-udm/lib/python*/site-packages/systemimage
022
=== added file 'coverage-udm.ini'
--- coverage-udm.ini 1970-01-01 00:00:00 +0000
+++ coverage-udm.ini 2015-05-20 14:55:53 +0000
@@ -0,0 +1,21 @@
1[run]
2branch = true
3parallel = true
4omit =
5 setup*
6 systemimage/data/*
7 systemimage/docs/*
8 systemimage/testing/*
9 systemimage/tests/*
10 systemimage/curl.py
11 /usr/lib/*
12 .tox/coverage-curl/lib/python3.4/distutils/*
13 .tox/coverage-curl/lib/python3.4/site-packages/pkg_resources*
14 .tox/coverage-udm/lib/python3.4/distutils/*
15 .tox/coverage-udm/lib/python3.4/site-packages/pkg_resources*
16
17[paths]
18source =
19 systemimage
20 .tox/coverage-curl/lib/python*/site-packages/systemimage
21 .tox/coverage-udm/lib/python*/site-packages/systemimage
022
=== removed file 'coverage.ini'
--- coverage.ini 2014-09-17 02:58:58 +0000
+++ coverage.ini 1970-01-01 00:00:00 +0000
@@ -1,15 +0,0 @@
1[run]
2branch = true
3parallel = true
4omit =
5 setup*
6 systemimage/data/*
7 systemimage/docs/*
8 systemimage/testing/*
9 systemimage/tests/*
10 /usr/lib/*
11
12[paths]
13source =
14 systemimage
15 .tox/coverage/lib/python*/site-packages/systemimage
160
=== modified file 'dbus-manpage.rst'
--- dbus-manpage.rst 2014-09-26 14:36:34 +0000
+++ dbus-manpage.rst 2015-05-20 14:55:53 +0000
@@ -7,9 +7,9 @@
7-----------------------------------------7-----------------------------------------
88
9:Author: Barry Warsaw <barry@ubuntu.com>9:Author: Barry Warsaw <barry@ubuntu.com>
10:Date: 2014-07-1510:Date: 2015-01-15
11:Copyright: 2013-2014 Canonical Ltd.11:Copyright: 2013-2015 Canonical Ltd.
12:Version: 2.312:Version: 3.0
13:Manual section: 813:Manual section: 8
1414
1515
@@ -42,10 +42,13 @@
42 enabled. With two ``-v`` (or ``-vv``), logging both to the console and to42 enabled. With two ``-v`` (or ``-vv``), logging both to the console and to
43 the log file are output at ``DEBUG`` level.43 the log file are output at ``DEBUG`` level.
4444
45-C FILE, --config FILE45-C DIR, --config DIR
46 Use the given configuration file, otherwise use the default. The program46 Use the given configuration directory, otherwise use the system default.
47 will optionally also read a ``channel.ini`` file in the same directory as47 The program will read all the files in this directory that begin with a
48 ``FILE``.48 number, followed by an underscore, and ending in ``.ini``
49 (e.g. ``03_myconfig.ini``). The files are read in sorted numerical order
50 from lowest prefix number to highest, with later configuration files able
51 to override any variable in any section.
4952
5053
51D-BUS API54D-BUS API
@@ -94,11 +97,12 @@
9497
95``ApplyUpdate()``98``ApplyUpdate()``
96 This is an **asynchronous** call used to apply a previously downloaded99 This is an **asynchronous** call used to apply a previously downloaded
97 update and initiate a reboot to apply the update. It is a no-op if no new100 update. After the update has been applied, an ``Applied`` signal is
98 update has been downloaded. Just before the device reboots, a101 sent. Some devices require a reboot in order to apply the update, and
99 ``Rebooting`` signal is sent, although the timing of this signal both102 such devices may also issue a ``Rebooting`` signal. However, on devices
100 being sent and received depends on how quickly the device is shut down for103 which require a reboot, the timing and emission of both the ``Applied``
101 reboot.104 and ``Rebooting`` signals are in a race condition with system shutdown,
105 and may not occur.
102106
103``CancelUpdate()``107``CancelUpdate()``
104 This is a **synchronous** call to cancel any update check or download in108 This is a **synchronous** call to cancel any update check or download in
@@ -143,12 +147,21 @@
143 * *version_detail* - A string containing a comma-separated list of147 * *version_detail* - A string containing a comma-separated list of
144 key-value pairs providing additional component version details,148 key-value pairs providing additional component version details,
145 e.g. "ubuntu=123,mako=456,custom=789".149 e.g. "ubuntu=123,mako=456,custom=789".
150 * *target_version_detail* - Like *version_detail* but contains the
151 information from the server. If an update is known to be available,
152 this will be taken from ``index.json`` file's image specification, for
153 the image that the upgrade will leave the device at. If no update is
154 available this will be identical to *version_detail*. If no
155 `CheckForUpdate()` as been previously performed, then the
156 *target_version_detail* will be the empty string.
146 * *last_check_date* - The last time a ``CheckForUpdate()`` call was157 * *last_check_date* - The last time a ``CheckForUpdate()`` call was
147 performed.158 performed.
148159
149 *New in system-image 2.3*160 *New in system-image 2.3*
150161
151 *New in system-image 2.5: target_build_number*162 *New in system-image 2.5: target_build_number was added.*
163
164 *New in system-image 3.0: target_version_detail was added.*
152165
153``FactoryReset()``166``FactoryReset()``
154 This is a **synchronous** call which wipes the data partition and issue a167 This is a **synchronous** call which wipes the data partition and issue a
@@ -157,6 +170,13 @@
157170
158 *New in system-image 2.3*.171 *New in system-image 2.3*.
159172
173``ProductionReset()``
174 This is a **synchronous** call which wipes the data partition, sets a flag
175 for factory wipe (used in production), and issue a reboot to recovery.
176 A ``Rebooting`` signal may be sent, depending on timing.
177
178 *New in system-image 3.0*.
179
160``SetSetting(key, value)``180``SetSetting(key, value)``
161 This is a **synchronous** call to write or update a setting. ``key`` and181 This is a **synchronous** call to write or update a setting. ``key`` and
162 ``value`` are strings. While any key/value pair may be set, some keys182 ``value`` are strings. While any key/value pair may be set, some keys
@@ -265,15 +285,20 @@
265 * **last_reason** - A string containing the reason for why this updated285 * **last_reason** - A string containing the reason for why this updated
266 failed.286 failed.
267287
288``Applied(status)``
289 Sent in response to an ``ApplyUpdate()`` call. See the timing caveats for
290 that method. **New in system-image 3.0**
291
292 * **status** - A boolean indicating whether an update has been applied or
293 not.
294
268``Rebooting(status)``295``Rebooting(status)``
269 Sent just before the device reboots. Because the system is in the process296 On devices which require a reboot in order to apply an update, this signal
270 of being rebooted, clients may or may not receive this signal.297 may be sent in response to an ``ApplyUpdate()`` call. See the timing
298 caveats for that method.
271299
272 * **status** - A boolean indicating whether the application of the update300 * **status** - A boolean indicating whether the device has initiated a
273 is successful or not. Generally, when status is true you won't ever301 reboot sequence or not.
274 receive the signal because the device will be rebooting. When status is
275 false it means the application of the update or reboot failed for some
276 reason.
277302
278``SettingChanged(key, value)``303``SettingChanged(key, value)``
279 Sent when a setting is changed. This signal is not sent if the new value304 Sent when a setting is changed. This signal is not sent if the new value
@@ -286,7 +311,7 @@
286Additional API details311Additional API details
287----------------------312----------------------
288313
289The ``SetSettings()`` call takes a key string and a value string. The314The ``SetSetting()`` call takes a key string and a value string. The
290following keys are predefined.315following keys are predefined.
291316
292 * *min_battery* - The minimum battery strength which will allow downloads317 * *min_battery* - The minimum battery strength which will allow downloads
@@ -311,12 +336,8 @@
311FILES336FILES
312=====337=====
313338
314/etc/system-image/client.ini339/etc/system-image/[0-9]+*.ini
315 Default configuration file.340 Default configuration files.
316
317/etc/system-image/channel.ini
318 Optional configuration file overrides (for the ``[service]`` section
319 only).
320341
321/etc/dbus-1/system.d/com.canonical.SystemImage.conf342/etc/dbus-1/system.d/com.canonical.SystemImage.conf
322 DBus service permissions file.343 DBus service permissions file.
@@ -328,6 +349,7 @@
328SEE ALSO349SEE ALSO
329========350========
330351
331client.ini(5), system-image-cli(1)352system-image.ini(5), system-image-cli(1)
353
332354
333.. _`ISO 8601`: http://en.wikipedia.org/wiki/ISO_8601355.. _`ISO 8601`: http://en.wikipedia.org/wiki/ISO_8601
334356
=== modified file 'debian/changelog'
--- debian/changelog 2014-09-29 19:02:48 +0000
+++ debian/changelog 2015-05-20 14:55:53 +0000
@@ -1,3 +1,80 @@
1system-image (3.0-0ubuntu2) UNRELEASED; urgency=medium
2
3 * New upstream release.
4 - LP: #1374459 - Support a built-in PyCURL-based downloader in
5 addition to the traditional ubuntu-download-manager (over D-BUS)
6 downloader. Auto-detects which downloader to use based on whether
7 udm is available on the system bus, pycurl is importable, and the
8 setting of the SYSTEMIMAGE_PYCURL environment variable. Initial
9 contribution by Michael Vogt.
10 - LP: #1384859 - Support alternative machine-id files as fall backs if
11 the D-Bus file does not exist. Specifically, add systemd's
12 /etc/machine-id to the list. Initial contribution by Michael Vogt.
13 - LP: #1373467 - Support multiple configuration files, as in a
14 `config.d` directory. Now, configuration files are named
15 `NN_whatever.ini` where "NN" must be a numeric prefix. Files are
16 loaded in sorted numeric order, with later files overriding newer
17 files. Support for both the `client.ini` and `channel.ini` files has
18 been removed.
19 - The `[system]build_file` variable has been removed. Build number
20 information now must come from the `.ini` files, and last update
21 date comes from the newest `.ini` file loaded.
22 - The `-C` command line option now takes a path to the configuration
23 directory.
24 - LP: #1412698 - Reworked the checking and downloading locks/flags to
25 so that they will work better with configuration reloading.
26 - LP: #1377312 - Support for the `/etc/ubuntu-build` file has been
27 removed. The build number now comes from the configuration files.
28 - LP: #1377184 - Move the `archive-master.tar.xz` file to
29 `/usr/share/system-image` for better FHS compliance.
30 - LP: #1381538 - Since devices do not always reboot to apply changes,
31 the `[hooks]update` variable has been renamed to `[hooks]apply`.
32 - LP: #1333414 - For testing purposes only, `system-image-cli` now
33 supports an undocumented command line switch
34 `--skip-gpg-verification`. Originally given by Jani Monoses.
35 - LP: #1417176 - A new D-Bus signal `Applied(bool)` is added, which is
36 returned in response to the `ApplyUpdate()` asynchronous method
37 call. For devices which do not need to reboot in order to apply the
38 update, this is the only signal you will get. If your device needs
39 to reboot you will also receive the `Rebooting(bool)` command as
40 with earlier versions. The semantics of the flag argument are the
41 same in both cases, as are the race timing issues inherent in these
42 signals. See the `system-image-dbus(8)` manpage for details.
43 - As part of LP: #1417176, the `--no-reboot` switch for
44 `system-image-cli(1)` has been deprecated. Use `--no-apply` instead
45 (`-g` is still the shortcut).
46 - LP: #1419027 - Support production factory resets. `system-image-cli
47 --production-reset` and a new D-Bus API method `ProductionReset()`
48 are added. Given by Ricardo Salveti.
49 - LP: #1399687 - A new key, `target_version_detail` has been added to
50 the dictionary returned by the `.Information()` D-Bus method.
51 - LP: #1387719 - The `User-Agent` HTTP header now also includes device
52 and channel names.
53 - LP: #1423622 - Added `--progress` flag to `system-image-cli` for
54 specifying methods for reporting progress. Current available values
55 are: `dots` (compatible with system-image 2.5), `logfile`
56 (compatible with system-image 2.5's `--verbose` flag), and `json`
57 for JSON records on stdout.
58 - Support for the `SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS` environment
59 variable has been removed.
60 - LP: #1448153 - Fix `system-image-cli --list-channels`.
61 * d/rules:
62 - Run both the cURL and UDM based tests.
63 - Run tests with more verbosity.
64 - Install the archive-master keyring files to /usr/share instead of
65 /etc for better FHS compliance. (LP: #1377184)
66 * d/control:
67 - Add python3-pycurl to Build-Depends.
68 - Bump Standards-Version to 3.9.6 with no other changes necessary.
69 - system-image-common now depends on
70 `ubuntu-download-manager | python3-pycurl` so that UDM doesn't need to
71 be pulled in for snappy. (LP: #1431696)
72 * d/tests/control: Disable DEP-8 "smoketests" which try to access
73 external resources. This is now prohibited by policy for
74 pocket-promotion tests. (LP: #1457070)
75
76 -- Barry Warsaw <barry@ubuntu.com> Wed, 20 May 2015 10:46:17 -0400
77
1system-image (2.5-0ubuntu1) utopic; urgency=medium78system-image (2.5-0ubuntu1) utopic; urgency=medium
279
3 [ Barry Warsaw ]80 [ Barry Warsaw ]
481
=== modified file 'debian/control'
--- debian/control 2014-09-09 17:27:17 +0000
+++ debian/control 2015-05-20 14:55:53 +0000
@@ -16,10 +16,11 @@
16 python3-nose2,16 python3-nose2,
17 python3-pkg-resources,17 python3-pkg-resources,
18 python3-psutil,18 python3-psutil,
19 python3-pycurl,
19 python3-setuptools,20 python3-setuptools,
20 python3-xdg,21 python3-xdg,
21 ubuntu-download-manager22 ubuntu-download-manager
22Standards-Version: 3.9.523Standards-Version: 3.9.6
23XS-Testsuite: autopkgtest24XS-Testsuite: autopkgtest
24Vcs-Bzr: https://code.launchpad.net/~ubuntu-managed-branches/ubuntu-system-image/system-image25Vcs-Bzr: https://code.launchpad.net/~ubuntu-managed-branches/ubuntu-system-image/system-image
25Vcs-Browser: http://bazaar.launchpad.net/~ubuntu-managed-branches/ubuntu-system-image/system-image/files26Vcs-Browser: http://bazaar.launchpad.net/~ubuntu-managed-branches/ubuntu-system-image/system-image/files
@@ -48,7 +49,7 @@
48 python3-gnupg,49 python3-gnupg,
49 python3-pkg-resources,50 python3-pkg-resources,
50 python3-xdg,51 python3-xdg,
51 ubuntu-download-manager,52 ubuntu-download-manager | python3-pycurl,
52 ${misc:Depends},53 ${misc:Depends},
53 ${python3:Depends}54 ${python3:Depends}
54Description: Ubuntu system image updater55Description: Ubuntu system image updater
5556
=== modified file 'debian/rules'
--- debian/rules 2014-09-09 17:27:17 +0000
+++ debian/rules 2015-05-20 14:55:53 +0000
@@ -9,11 +9,18 @@
9 dh $@ --with python3 --buildsystem=pybuild9 dh $@ --with python3 --buildsystem=pybuild
1010
11override_dh_auto_test:11override_dh_auto_test:
12 unset http_proxy; unset https_proxy; \12 export http_proxy= ; \
13 export SYSTEMIMAGE_REACTOR_TIMEOUT=1200; \13 export https_proxy= ; \
14 export SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS=2; \14 export SYSTEMIMAGE_REACTOR_TIMEOUT=1200; \
15 PYBUILD_SYSTEM=custom \15 export SYSTEMIMAGE_PYCURL=1; \
16 PYBUILD_TEST_ARGS="{interpreter} -m nose2 -v" dh_auto_test16 PYBUILD_SYSTEM=custom \
17 PYBUILD_TEST_ARGS="{interpreter} -m nose2 -vv" dh_auto_test
18 export http_proxy= ; \
19 export https_proxy= ; \
20 export SYSTEMIMAGE_REACTOR_TIMEOUT=1200; \
21 export SYSTEMIMAGE_PYCURL=0; \
22 PYBUILD_SYSTEM=custom \
23 PYBUILD_TEST_ARGS="{interpreter} -m nose2 -vv" dh_auto_test
1724
18# pybuild can't yet handle Python 3 packages that don't start with "python3-".25# pybuild can't yet handle Python 3 packages that don't start with "python3-".
19# See bug #751908 - In the meantime, this override isn't perfect, but it gets26# See bug #751908 - In the meantime, this override isn't perfect, but it gets
@@ -30,11 +37,9 @@
30 usr/lib/python3.?/dist-packages/systemimage/testing37 usr/lib/python3.?/dist-packages/systemimage/testing
31 dh_install -p system-image-cli usr/bin/system-image-cli38 dh_install -p system-image-cli usr/bin/system-image-cli
32 dh_install -p system-image-common \39 dh_install -p system-image-common \
33 debian/archive-master.tar.xz etc/system-image40 debian/archive-master.tar.xz usr/share/system-image
34 dh_install -p system-image-common \41 dh_install -p system-image-common \
35 debian/archive-master.tar.xz.asc etc/system-image42 debian/archive-master.tar.xz.asc usr/share/system-image
36 dh_install -p system-image-common \
37 systemimage/data/client.ini etc/system-image
38 dh_install -p system-image-dbus usr/bin/system-image-dbus usr/sbin43 dh_install -p system-image-dbus usr/bin/system-image-dbus usr/sbin
39 dh_install -p system-image-dbus \44 dh_install -p system-image-dbus \
40 systemimage/data/com.canonical.SystemImage.service \45 systemimage/data/com.canonical.SystemImage.service \
4146
=== renamed file 'debian/tests/client.ini.in' => 'debian/tests/00_default.ini.in'
=== modified file 'debian/tests/control'
--- debian/tests/control 2014-07-23 22:51:19 +0000
+++ debian/tests/control 2015-05-20 14:55:53 +0000
@@ -1,11 +1,11 @@
1Tests: smoketest1#Tests: smoketest
2Restrictions: isolation-container2#Restrictions: isolation-container
3Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg3#Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-pycurl
44
5Tests: smoketest-noreboot5#Tests: smoketest-noreboot
6Restrictions: isolation-container allow-stderr6#Restrictions: isolation-container allow-stderr
7Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg7#Depends: system-image-common, system-image-cli, system-image-dbus, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-pycurl
88
9Tests: dryrun9Tests: dryrun
10Restrictions: allow-stderr10Restrictions: allow-stderr
11Depends: @, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-setuptools, python3-nose211Depends: @, ubuntu-download-manager, dbus, dbus-x11, python3-psutil, python3-xdg, python3-setuptools, python3-nose2, python3-pycurl
1212
=== modified file 'debian/tests/dryrun'
--- debian/tests/dryrun 2014-07-18 16:32:44 +0000
+++ debian/tests/dryrun 2015-05-20 14:55:53 +0000
@@ -5,7 +5,7 @@
5# require network access, so it is compatible with less isolated (but also5# require network access, so it is compatible with less isolated (but also
6# lighter weight) containers such as schroot.6# lighter weight) containers such as schroot.
7#7#
8# Copyright (C) 2014 Canonical Ltd.8# Copyright (C) 2014-2015 Canonical Ltd.
9# Author: Barry Warsaw <barry@ubuntu.com>9# Author: Barry Warsaw <barry@ubuntu.com>
1010
11python3 -m nose2 -vv -P TestCLIMainDryRun11python3 -m nose2 -vv -P TestCLIMainDryRun
1212
=== modified file 'debian/tests/prep.py'
--- debian/tests/prep.py 2013-12-13 13:55:51 +0000
+++ debian/tests/prep.py 2015-05-20 14:55:53 +0000
@@ -1,5 +1,8 @@
1#!/usr/bin/python31#!/usr/bin/python3
22
3# Copyright (C) 2013-2015 Canonical Ltd.
4# Author: Barry Warsaw <barry@ubuntu.com>
5
3import os6import os
47
5tmpdir = os.environ['ADTTMP']8tmpdir = os.environ['ADTTMP']
@@ -8,15 +11,19 @@
8os.makedirs(os.path.join(tmpdir, 'android'), exist_ok=True)11os.makedirs(os.path.join(tmpdir, 'android'), exist_ok=True)
9os.makedirs(os.path.join(tmpdir, 'ubuntu'), exist_ok=True)12os.makedirs(os.path.join(tmpdir, 'ubuntu'), exist_ok=True)
1013
14config_d = os.path.join(tmpdir, 'config.d')
15os.makedirs(config_d, exist_ok=True)
16
11substitutions = dict(17substitutions = dict(
12 TMPDIR=tmpdir,18 TMPDIR=tmpdir,
13 ARTIFACTS=artifacts,19 ARTIFACTS=artifacts,
14 )20 )
1521
16with open('debian/tests/client.ini.in', encoding='utf-8') as fp:22with open('debian/tests/00_default.ini.in', encoding='utf-8') as fp:
17 ini_template = fp.read()23 ini_template = fp.read()
1824
19ini_contents = ini_template.format(**substitutions)25ini_contents = ini_template.format(**substitutions)
2026
21with open(os.path.join(tmpdir, 'client.ini'), 'w', encoding='utf-8') as fp:27default_ini = os.path.join(config_d, '00_default.ini')
28with open(default_ini, 'w', encoding='utf-8') as fp:
22 fp.write(ini_contents)29 fp.write(ini_contents)
2330
=== modified file 'debian/tests/smoketest'
--- debian/tests/smoketest 2014-07-18 16:32:44 +0000
+++ debian/tests/smoketest 2015-05-20 14:55:53 +0000
@@ -5,9 +5,9 @@
5# isolation-container restricted, so it requires an isolated test container5# isolation-container restricted, so it requires an isolated test container
6# like QEMU.6# like QEMU.
7#7#
8# Copyright (C) 2013-2014 Canonical Ltd.8# Copyright (C) 2013-2015 Canonical Ltd.
9# Author: Barry Warsaw <barry@ubuntu.com>9# Author: Barry Warsaw <barry@ubuntu.com>
1010
11set -e11set -e
12python3 debian/tests/prep.py12python3 debian/tests/prep.py
13system-image-cli -C $ADTTMP/client.ini -d mako -c devel -b 0 --dry-run13system-image-cli -C $ADTTMP/config.d -d mako -c devel -b 0 --dry-run
1414
=== modified file 'debian/tests/smoketest-noreboot'
--- debian/tests/smoketest-noreboot 2014-07-23 22:51:19 +0000
+++ debian/tests/smoketest-noreboot 2015-05-20 14:55:53 +0000
@@ -7,9 +7,9 @@
7#7#
8# This is like smoketest except that it does a full download.8# This is like smoketest except that it does a full download.
9#9#
10# Copyright (C) 2013-2014 Canonical Ltd.10# Copyright (C) 2013-2015 Canonical Ltd.
11# Author: Barry Warsaw <barry@ubuntu.com>11# Author: Barry Warsaw <barry@ubuntu.com>
1212
13set -e13set -e
14python3 debian/tests/prep.py14python3 debian/tests/prep.py
15system-image-cli -C $ADTTMP/client.ini -d mako -c devel -b 0 --no-reboot -v15system-image-cli -C $ADTTMP/config.d -d mako -c devel -b 0 --no-reboot -v
1616
=== modified file 'debian/tests/unittests'
--- debian/tests/unittests 2013-12-13 13:55:51 +0000
+++ debian/tests/unittests 2015-05-20 14:55:53 +0000
@@ -2,7 +2,7 @@
2#2#
3# autopkgtest check: Run tox against the built package.3# autopkgtest check: Run tox against the built package.
4#4#
5# Copyright (C) 2013 Canonical Ltd.5# Copyright (C) 2013-2015 Canonical Ltd.
6# Author: Barry Warsaw <barry@ubuntu.com>6# Author: Barry Warsaw <barry@ubuntu.com>
77
8set -e8set -e
99
=== modified file 'ini-manpage.rst'
--- ini-manpage.rst 2014-09-17 13:41:31 +0000
+++ ini-manpage.rst 2015-05-20 14:55:53 +0000
@@ -1,39 +1,43 @@
1==========1================
2client.ini2system-image.ini
3==========3================
44
55
6-----------------------------------------------6------------------------------------------------
7Ubuntu System Image Upgrader configuration file7Ubuntu System Image Upgrader configuration files
8-----------------------------------------------8------------------------------------------------
99
10:Author: Barry Warsaw <barry@ubuntu.com>10:Author: Barry Warsaw <barry@ubuntu.com>
11:Date: 2014-09-1111:Date: 2015-01-15
12:Copyright: 2013-2014 Canonical Ltd.12:Copyright: 2013-2015 Canonical Ltd.
13:Version: 2.413:Version: 3.0
14:Manual section: 514:Manual section: 5
1515
1616
17DESCRIPTION17DESCRIPTION
18===========18===========
1919
20``/etc/system-image/client.ini`` is the configuration file for the system20``/etc/system-image/config.d`` is the default configuration directory for the
21image upgrader. It is an ini-style configuration file with sections that21system image upgrader. It contains ini-style configuration files with
22define the service to connect to, as well as local system resources.22sections that define the service to connect to, as well as local system
23Generally, the options never need to be changed.23resources. Generally, the options never need to be changed.
2424
25The system image upgrader will also optionally read a25The system image upgrader will read all files in this directory that start
26``/etc/system-image/channel.ini`` file with the same format as ``client.ini``.26with a numeric prefix, followed by an underscore, and then any alphanumeric
27This file should only contain a ``[service]`` section for overriding in the27suffix, ending in ``.ini``. E.g. ``07_myconfig.ini``.
28``client.ini`` file. All other sections are ignored.28
29The files are read in sorted numerical order, from lowest prefix number to
30highest, with later configuration files able to override any variable in any
31section.
2932
3033
31SYNTAX34SYNTAX
32======35======
3336
34Sections are delimited by square brackets, e.g. ``[service]``. Variables37Sections in the ``.ini`` files are delimited by square brackets,
35inside the service separate the variable name and value by a colon. Blank38e.g. ``[service]``. Variables inside the service separate the variable name
36lines and lines that start with a ``#`` are ignored.39and value by a colon. Blank lines and lines that start with a ``#`` are
40ignored.
3741
3842
39THE SERVICE SECTION43THE SERVICE SECTION
@@ -82,10 +86,6 @@
8286
83This section contains the following variables:87This section contains the following variables:
8488
85build_file
86 The file on the local file system containing the system's current build
87 number.
88
89tempdir89tempdir
90 The base temporary directory on the local file system. When any of the90 The base temporary directory on the local file system. When any of the
91 system-image processes run, a secure subdirectory inside `tempdir` will be91 system-image processes run, a secure subdirectory inside `tempdir` will be
@@ -181,9 +181,11 @@
181 The Python import path to the class implementing the upgrade scoring181 The Python import path to the class implementing the upgrade scoring
182 algorithm.182 algorithm.
183183
184reboot184apply
185 The Python import path to the class that implements the system reboot185 The Python import path to the class that implements the mechanism for
186 command.186 applying the update. This often reboots the device.
187
188 *New in system-image 3.0: ``reboot`` was renamed to ``apply``*
187189
188190
189THE DBUS SECTION191THE DBUS SECTION
@@ -204,6 +206,7 @@
204206
205system-image-cli(1)207system-image-cli(1)
206208
209
207[1]: https://wiki.ubuntu.com/ImageBasedUpgrades/Server210[1]: https://wiki.ubuntu.com/ImageBasedUpgrades/Server
208211
209[2]: https://wiki.ubuntu.com/ImageBasedUpgrades/GPG212[2]: https://wiki.ubuntu.com/ImageBasedUpgrades/GPG
210213
=== modified file 'setup.cfg'
--- setup.cfg 2014-09-26 14:36:34 +0000
+++ setup.cfg 2015-05-20 14:55:53 +0000
@@ -4,7 +4,7 @@
4logging-filter = systemimage4logging-filter = systemimage
55
6[egg_info]6[egg_info]
7tag_svn_revision = 0
8tag_build = 7tag_build =
9tag_date = 08tag_date = 0
9tag_svn_revision = 0
1010
1111
=== modified file 'setup.py'
--- setup.py 2014-02-20 23:03:24 +0000
+++ setup.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
55
=== modified file 'system_image.egg-info/PKG-INFO'
--- system_image.egg-info/PKG-INFO 2014-09-26 14:36:34 +0000
+++ system_image.egg-info/PKG-INFO 2015-05-20 14:55:53 +0000
@@ -1,6 +1,6 @@
1Metadata-Version: 1.01Metadata-Version: 1.0
2Name: system-image2Name: system-image
3Version: 2.53Version: 3.0
4Summary: Ubuntu System Image Based Upgrades4Summary: Ubuntu System Image Based Upgrades
5Home-page: UNKNOWN5Home-page: UNKNOWN
6Author: Barry Warsaw6Author: Barry Warsaw
77
=== modified file 'system_image.egg-info/SOURCES.txt'
--- system_image.egg-info/SOURCES.txt 2014-09-26 14:36:34 +0000
+++ system_image.egg-info/SOURCES.txt 2015-05-20 14:55:53 +0000
@@ -2,7 +2,8 @@
2NEWS.rst2NEWS.rst
3README.rst3README.rst
4cli-manpage.rst4cli-manpage.rst
5coverage.ini5coverage-curl.ini
6coverage-udm.ini
6dbus-manpage.rst7dbus-manpage.rst
7ini-manpage.rst8ini-manpage.rst
8setup.cfg9setup.cfg
@@ -17,10 +18,12 @@
17system_image.egg-info/top_level.txt18system_image.egg-info/top_level.txt
18systemimage/__init__.py19systemimage/__init__.py
19systemimage/api.py20systemimage/api.py
21systemimage/apply.py
20systemimage/bag.py22systemimage/bag.py
21systemimage/candidates.py23systemimage/candidates.py
22systemimage/channel.py24systemimage/channel.py
23systemimage/config.py25systemimage/config.py
26systemimage/curl.py
24systemimage/dbus.py27systemimage/dbus.py
25systemimage/device.py28systemimage/device.py
26systemimage/download.py29systemimage/download.py
@@ -32,14 +35,13 @@
32systemimage/logging.py35systemimage/logging.py
33systemimage/main.py36systemimage/main.py
34systemimage/reactor.py37systemimage/reactor.py
35systemimage/reboot.py
36systemimage/scores.py38systemimage/scores.py
37systemimage/service.py39systemimage/service.py
38systemimage/settings.py40systemimage/settings.py
39systemimage/state.py41systemimage/state.py
42systemimage/udm.py
40systemimage/version.txt43systemimage/version.txt
41systemimage/data/__init__.py44systemimage/data/__init__.py
42systemimage/data/client.ini
43systemimage/data/com.canonical.SystemImage.conf45systemimage/data/com.canonical.SystemImage.conf
44systemimage/data/com.canonical.SystemImage.service46systemimage/data/com.canonical.SystemImage.service
45systemimage/docs/__init__.py47systemimage/docs/__init__.py
@@ -70,77 +72,121 @@
70systemimage/tests/test_settings.py72systemimage/tests/test_settings.py
71systemimage/tests/test_state.py73systemimage/tests/test_state.py
72systemimage/tests/test_winner.py74systemimage/tests/test_winner.py
75systemimage/tests/data/00.ini
76systemimage/tests/data/01.ini
73systemimage/tests/data/__init__.py77systemimage/tests/data/__init__.py
78systemimage/tests/data/api.channels_01.json
79systemimage/tests/data/api.index_01.json
80systemimage/tests/data/api.index_02.json
81systemimage/tests/data/api.index_03.json
74systemimage/tests/data/archive-master.gpg82systemimage/tests/data/archive-master.gpg
75systemimage/tests/data/bad_cert.pem83systemimage/tests/data/bad_cert.pem
76systemimage/tests/data/bad_key.pem84systemimage/tests/data/bad_key.pem
85systemimage/tests/data/candidates.index_01.json
86systemimage/tests/data/candidates.index_02.json
87systemimage/tests/data/candidates.index_03.json
88systemimage/tests/data/candidates.index_04.json
89systemimage/tests/data/candidates.index_05.json
90systemimage/tests/data/candidates.index_06.json
91systemimage/tests/data/candidates.index_07.json
92systemimage/tests/data/candidates.index_08.json
93systemimage/tests/data/candidates.index_09.json
94systemimage/tests/data/candidates.index_10.json
95systemimage/tests/data/candidates.index_11.json
96systemimage/tests/data/candidates.index_12.json
97systemimage/tests/data/candidates.index_13.json
77systemimage/tests/data/cert.pem98systemimage/tests/data/cert.pem
78systemimage/tests/data/channel_01.ini99systemimage/tests/data/channel.channels_01.json
79systemimage/tests/data/channel_02.ini100systemimage/tests/data/channel.channels_02.json
80systemimage/tests/data/channel_03.ini101systemimage/tests/data/channel.channels_03.json
81systemimage/tests/data/channel_04.ini102systemimage/tests/data/channel.channels_04.json
82systemimage/tests/data/channel_05.ini103systemimage/tests/data/channel.channels_05.json
83systemimage/tests/data/channel_06.ini
84systemimage/tests/data/channel_07.ini
85systemimage/tests/data/channels_01.json
86systemimage/tests/data/channels_02.json
87systemimage/tests/data/channels_03.json
88systemimage/tests/data/channels_04.json
89systemimage/tests/data/channels_05.json
90systemimage/tests/data/channels_06.json
91systemimage/tests/data/channels_07.json
92systemimage/tests/data/channels_08.json
93systemimage/tests/data/channels_09.json
94systemimage/tests/data/channels_10.json
95systemimage/tests/data/channels_11.json
96systemimage/tests/data/com.canonical.SystemImage.service.in104systemimage/tests/data/com.canonical.SystemImage.service.in
97systemimage/tests/data/com.canonical.applications.Downloader.service.in105systemimage/tests/data/com.canonical.applications.Downloader.service.in
98systemimage/tests/data/config_00.ini106systemimage/tests/data/config.config_01.ini
99systemimage/tests/data/config_01.ini107systemimage/tests/data/config.config_02.ini
100systemimage/tests/data/config_02.ini108systemimage/tests/data/config.config_03.ini
101systemimage/tests/data/config_03.ini109systemimage/tests/data/config.config_04.ini
102systemimage/tests/data/config_04.ini110systemimage/tests/data/config.config_05.ini
103systemimage/tests/data/config_05.ini111systemimage/tests/data/config.config_06.ini
104systemimage/tests/data/config_06.ini112systemimage/tests/data/config.config_07.ini
105systemimage/tests/data/config_07.ini113systemimage/tests/data/config.config_08.ini
106systemimage/tests/data/config_08.ini114systemimage/tests/data/config.config_09.ini
107systemimage/tests/data/config_09.ini115systemimage/tests/data/config.config_10.ini
108systemimage/tests/data/config_10.ini116systemimage/tests/data/config.config_11.ini
109systemimage/tests/data/dbus-system.conf.in117systemimage/tests/data/dbus-system.conf.in
118systemimage/tests/data/dbus.channels_01.json
119systemimage/tests/data/dbus.index_01.json
120systemimage/tests/data/dbus.index_02.json
121systemimage/tests/data/dbus.index_03.json
122systemimage/tests/data/dbus.index_04.json
123systemimage/tests/data/dbus.index_05.json
124systemimage/tests/data/dbus.index_06.json
110systemimage/tests/data/device-signing.gpg125systemimage/tests/data/device-signing.gpg
126systemimage/tests/data/download.index_01.json
111systemimage/tests/data/expired_cert.pem127systemimage/tests/data/expired_cert.pem
112systemimage/tests/data/expired_key.pem128systemimage/tests/data/expired_key.pem
129systemimage/tests/data/gpg.channels_01.json
130systemimage/tests/data/helpers.config_01.ini
131systemimage/tests/data/helpers.config_02.ini
113systemimage/tests/data/image-master.gpg132systemimage/tests/data/image-master.gpg
114systemimage/tests/data/image-signing.gpg133systemimage/tests/data/image-signing.gpg
115systemimage/tests/data/index_01.json134systemimage/tests/data/index.channels_01.json
116systemimage/tests/data/index_02.json135systemimage/tests/data/index.channels_02.json
117systemimage/tests/data/index_03.json136systemimage/tests/data/index.channels_03.json
118systemimage/tests/data/index_04.json137systemimage/tests/data/index.channels_04.json
119systemimage/tests/data/index_05.json138systemimage/tests/data/index.channels_05.json
120systemimage/tests/data/index_06.json139systemimage/tests/data/index.index_01.json
121systemimage/tests/data/index_07.json140systemimage/tests/data/index.index_02.json
122systemimage/tests/data/index_08.json141systemimage/tests/data/index.index_03.json
123systemimage/tests/data/index_09.json142systemimage/tests/data/index.index_04.json
124systemimage/tests/data/index_10.json143systemimage/tests/data/index.index_05.json
125systemimage/tests/data/index_11.json
126systemimage/tests/data/index_12.json
127systemimage/tests/data/index_13.json
128systemimage/tests/data/index_14.json
129systemimage/tests/data/index_15.json
130systemimage/tests/data/index_16.json
131systemimage/tests/data/index_17.json
132systemimage/tests/data/index_18.json
133systemimage/tests/data/index_19.json
134systemimage/tests/data/index_20.json
135systemimage/tests/data/index_21.json
136systemimage/tests/data/index_22.json
137systemimage/tests/data/index_23.json
138systemimage/tests/data/index_24.json
139systemimage/tests/data/index_25.json
140systemimage/tests/data/key.pem144systemimage/tests/data/key.pem
145systemimage/tests/data/main.channels_01.json
146systemimage/tests/data/main.channels_02.json
147systemimage/tests/data/main.channels_03.json
148systemimage/tests/data/main.config_01.ini
149systemimage/tests/data/main.config_02.ini
150systemimage/tests/data/main.config_03.ini
151systemimage/tests/data/main.config_04.ini
152systemimage/tests/data/main.config_05.ini
153systemimage/tests/data/main.config_07.ini
154systemimage/tests/data/main.index_01.json
155systemimage/tests/data/main.index_02.json
156systemimage/tests/data/main.index_03.json
157systemimage/tests/data/main.index_04.json
158systemimage/tests/data/main.index_05.json
141systemimage/tests/data/master-secring.gpg159systemimage/tests/data/master-secring.gpg
142systemimage/tests/data/nasty_cert.pem160systemimage/tests/data/nasty_cert.pem
143systemimage/tests/data/nasty_key.pem161systemimage/tests/data/nasty_key.pem
162systemimage/tests/data/scores.index_01.json
163systemimage/tests/data/scores.index_02.json
164systemimage/tests/data/scores.index_03.json
165systemimage/tests/data/scores.index_04.json
166systemimage/tests/data/scores.index_05.json
167systemimage/tests/data/scores.index_06.json
168systemimage/tests/data/scores.index_07.json
144systemimage/tests/data/spare.gpg169systemimage/tests/data/spare.gpg
145systemimage/tests/data/sprint_nexus7_index_01.json
146tools/demo.ini
147\ No newline at end of file170\ No newline at end of file
171systemimage/tests/data/state.channels_01.json
172systemimage/tests/data/state.channels_02.json
173systemimage/tests/data/state.channels_03.json
174systemimage/tests/data/state.channels_04.json
175systemimage/tests/data/state.channels_05.json
176systemimage/tests/data/state.channels_06.json
177systemimage/tests/data/state.channels_07.json
178systemimage/tests/data/state.config_01.ini
179systemimage/tests/data/state.config_02.ini
180systemimage/tests/data/state.index_01.json
181systemimage/tests/data/state.index_02.json
182systemimage/tests/data/state.index_03.json
183systemimage/tests/data/state.index_04.json
184systemimage/tests/data/state.index_05.json
185systemimage/tests/data/state.index_06.json
186systemimage/tests/data/state.index_07.json
187systemimage/tests/data/state.index_08.json
188systemimage/tests/data/winner.channels_01.json
189systemimage/tests/data/winner.channels_02.json
190systemimage/tests/data/winner.index_01.json
191systemimage/tests/data/winner.index_02.json
192tools/demo.ini
193tools/runme.sh
148\ No newline at end of file194\ No newline at end of file
149195
=== modified file 'systemimage/api.py'
--- systemimage/api.py 2014-09-17 13:41:31 +0000
+++ systemimage/api.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -24,10 +24,9 @@
2424
25import logging25import logging
2626
27from systemimage.helpers import last_update_date27from systemimage.apply import factory_reset, production_reset
28from systemimage.reboot import factory_reset
29from systemimage.state import State28from systemimage.state import State
30from unittest.mock import patch29
3130
32log = logging.getLogger('systemimage')31log = logging.getLogger('systemimage')
3332
@@ -63,8 +62,12 @@
63 return ''62 return ''
6463
65 @property64 @property
66 def last_update_date(self):65 def version_detail(self):
67 return last_update_date()66 try:
67 return self._winners[-1].version_detail
68 except IndexError:
69 # No winners.
70 return ''
6871
6972
70class Mediator:73class Mediator:
@@ -115,13 +118,20 @@
115 def download(self):118 def download(self):
116 """Download the available update."""119 """Download the available update."""
117 # We only want callback progress during the actual download.120 # We only want callback progress during the actual download.
118 with patch.object(self._state.downloader, 'callback', self._callback):121 old_callbacks = self._state.downloader.callbacks[:]
119 self._state.run_until('reboot')122 try:
123 self._state.downloader.callbacks = [self._callback]
124 self._state.run_until('apply')
125 finally:
126 self._state.downloader.callbacks = old_callbacks
120127
121 def reboot(self):128 def apply(self):
122 """Issue the reboot."""129 """Apply the update."""
123 # Transition through all remaining states.130 # Transition through all remaining states.
124 list(self._state)131 list(self._state)
125132
126 def factory_reset(self):133 def factory_reset(self):
127 factory_reset()134 factory_reset()
135
136 def production_reset(self):
137 production_reset()
128138
=== renamed file 'systemimage/reboot.py' => 'systemimage/apply.py'
--- systemimage/reboot.py 2014-09-17 13:41:31 +0000
+++ systemimage/apply.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -16,9 +16,11 @@
16"""Reboot issuer."""16"""Reboot issuer."""
1717
18__all__ = [18__all__ = [
19 'BaseReboot',19 'BaseApply',
20 'Noop',
20 'Reboot',21 'Reboot',
21 'factory_reset',22 'factory_reset',
23 'production_reset',
22 ]24 ]
2325
2426
@@ -32,24 +34,34 @@
32log = logging.getLogger('systemimage')34log = logging.getLogger('systemimage')
3335
3436
35class BaseReboot:37class BaseApply:
36 """Common reboot actions."""38 """Common apply-the-update actions."""
3739
38 def reboot(self): # pragma: no cover40 def apply(self): # pragma: no cover
39 """Subclasses must override this."""41 """Subclasses must override this."""
40 raise NotImplementedError42 raise NotImplementedError
4143
4244
43class Reboot(BaseReboot):45class Reboot(BaseApply):
44 """Issue a standard reboot."""46 """Apply the update by rebooting the device."""
4547
46 def reboot(self):48 def apply(self):
47 try:49 try:
48 check_call('/sbin/reboot -f recovery'.split(),50 check_call('/sbin/reboot -f recovery'.split(),
49 universal_newlines=True)51 universal_newlines=True)
50 except CalledProcessError as error:52 except CalledProcessError as error:
51 log.exception('reboot exit status: {}'.format(error.returncode))53 log.exception('reboot exit status: {}'.format(error.returncode))
52 raise54 raise
55 # This code may or may not run. We're racing against the system
56 # reboot procedure.
57 config.dbus_service.Rebooting(True)
58
59
60class Noop(BaseApply):
61 """No-op apply, mostly for testing."""
62
63 def apply(self):
64 pass
5365
5466
55def factory_reset():67def factory_reset():
@@ -59,4 +71,15 @@
59 with atomic(command_file) as fp:71 with atomic(command_file) as fp:
60 print('format data', file=fp)72 print('format data', file=fp)
61 log.info('Performing a factory reset')73 log.info('Performing a factory reset')
62 config.hooks.reboot().reboot()74 config.hooks.apply().apply()
75
76
77def production_reset():
78 """Perform a production reset."""
79 command_file = os.path.join(
80 config.updater.cache_partition, 'ubuntu_command')
81 with atomic(command_file) as fp:
82 print('format data', file=fp)
83 print('enable factory_wipe', file=fp)
84 log.info('Performing a production factory reset')
85 config.hooks.apply().apply()
6386
=== modified file 'systemimage/bag.py'
--- systemimage/bag.py 2014-09-17 13:41:31 +0000
+++ systemimage/bag.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -42,6 +42,12 @@
4242
4343
44class Bag:44class Bag:
45 # NOTE: This class's methods share a namespace with the possible
46 # configuration variable names in the various sections. Thus no variable
47 # in any section can be named `update`, `keys`, or `get`. They also can't
48 # be named like any of the non-public methods, but that's usually not a
49 # problem. Ideally, we'd name the methods part of the reserved namespace,
50 # but it seems like a low tech debt for now.
45 def __init__(self, *, converters=None, **kws):51 def __init__(self, *, converters=None, **kws):
46 self._converters = make_converter(converters)52 self._converters = make_converter(converters)
47 self.__original__ = {}53 self.__original__ = {}
4854
=== modified file 'systemimage/candidates.py'
--- systemimage/candidates.py 2014-09-17 13:41:31 +0000
+++ systemimage/candidates.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
55
=== modified file 'systemimage/channel.py'
--- systemimage/channel.py 2014-09-17 13:41:31 +0000
+++ systemimage/channel.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
55
=== modified file 'systemimage/config.py'
--- systemimage/config.py 2014-09-17 13:41:31 +0000
+++ systemimage/config.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -17,7 +17,6 @@
1717
18__all__ = [18__all__ = [
19 'Configuration',19 'Configuration',
20 'DISABLED',
21 'config',20 'config',
22 ]21 ]
2322
@@ -27,79 +26,180 @@
2726
28from configparser import ConfigParser27from configparser import ConfigParser
29from contextlib import ExitStack28from contextlib import ExitStack
30from pkg_resources import resource_filename29from pathlib import Path
31from systemimage.bag import Bag30from systemimage.bag import Bag
32from systemimage.helpers import (31from systemimage.helpers import (
33 as_loglevel, as_object, as_timedelta, makedirs, temporary_directory)32 NO_PORT, as_loglevel, as_object, as_port, as_stripped, as_timedelta,
3433 makedirs, temporary_directory)
3534
36DISABLED = object()35
36SECTIONS = ('service', 'system', 'gpg', 'updater', 'hooks', 'dbus')
37USER_AGENT = ('Ubuntu System Image Upgrade Client: '
38 'device={0.device};channel={0.channel};build={0.build_number}')
3739
3840
39def expand_path(path):41def expand_path(path):
40 return os.path.abspath(os.path.expanduser(path))42 return os.path.abspath(os.path.expanduser(path))
4143
4244
43def port_value_converter(value):45class SafeConfigParser(ConfigParser):
44 if value.lower() in ('disabled', 'disable'):46 """Like ConfigParser, but with default empty sections.
45 return DISABLED47
46 result = int(value)48 This makes the **style of loading keys/values into the Bag objects a
47 if result < 0:49 little cleaner since it doesn't have to worry about KeyErrors when a
48 raise ValueError(value)50 configuration file doesn't contain a section, which is allowed.
49 return result51 """
5052
5153 def __init__(self, *args, **kws):
52def device_converter(value):54 super().__init__(args, **kws)
53 return value.strip()55 for section in SECTIONS:
56 self[section] = {}
5457
5558
56class Configuration:59class Configuration:
57 def __init__(self, ini_file=None):60 def __init__(self, directory=None):
58 # Defaults.61 self._set_defaults()
59 self.config_file = None62 # Because the configuration object is a global singleton, it makes for
60 self.service = Bag()63 # a convenient place to stash information used by widely separate
61 self.system = Bag()64 # components. For example, this is a placeholder for rendezvous
62 if ini_file is None:65 # between the downloader and the D-Bus service. When running under
63 ini_file = resource_filename('systemimage.data', 'client.ini')66 # D-Bus and we get a `paused` signal from the download manager, we need
64 self.load(ini_file)67 # this to plumb through an UpdatePaused signal to our clients. It
65 self._override = False68 # rather sucks that we need a global for this, but I can't get the
66 # 2013-10-14 BAW This is a placeholder for rendezvous between the69 # plumbing to work otherwise. This seems like the least horrible place
67 # downloader and the D-Bus service. When running udner D-Bus and we70 # to stash this global.
68 # get a `paused` signal from the download manager, we need this to
69 # plumb through an UpdatePaused signal to our clients. It rather
70 # sucks that we need a global for this, but I can't get the plumbing
71 # to work otherwise. This seems like the least horrible place to
72 # stash this global.
73 self.dbus_service = None71 self.dbus_service = None
74 # Cache/overrides.72 # This is used to plumb command line arguments from the main() to
73 # other parts of the system.
74 self.skip_gpg_verification = False
75 # Cache.
75 self._device = None76 self._device = None
76 self._build_number = None77 self._build_number = None
78 self.build_number_override = False
77 self._channel = None79 self._channel = None
80 # This is used only to override the phased percentage via command line
81 # and the property setter.
82 self._phase_override = None
78 self._tempdir = None83 self._tempdir = None
84 self.config_d = None
85 self.ini_files = []
86 self.http_base = None
87 self.https_base = None
88 if directory is not None:
89 self.load(directory)
90 self._calculate_http_bases()
79 self._resources = ExitStack()91 self._resources = ExitStack()
80 atexit.register(self._resources.close)92 atexit.register(self._resources.close)
8193
82 def load(self, path, *, override=False):94 def _set_defaults(self):
83 parser = ConfigParser()95 self.service = Bag(
84 files_read = parser.read(path)96 base='system-image.ubuntu.com',
85 if files_read != [path]:97 http_port=80,
86 raise FileNotFoundError(path)98 https_port=443,
87 self.config_file = path99 channel='daily',
88 self.service.update(converters=dict(http_port=port_value_converter,100 build_number=0,
89 https_port=port_value_converter,101 )
102 self.system = Bag(
103 timeout=as_timedelta('1h'),
104 tempdir='/tmp',
105 logfile='/var/log/system-image/client.log',
106 loglevel=as_loglevel('info'),
107 settings_db='/var/lib/system-image/settings.db',
108 )
109 self.gpg = Bag(
110 archive_master='/usr/share/system-image/archive-master.tar.xz',
111 image_master='/var/lib/system-image/keyrings/image-master.tar.xz',
112 image_signing=
113 '/var/lib/system-image/keyrings/image-signing.tar.xz',
114 device_signing=
115 '/var/lib/system-image/keyrings/device-signing.tar.xz',
116 )
117 self.updater = Bag(
118 cache_partition='/android/cache/recovery',
119 data_partition='/var/lib/system-image',
120 )
121 self.hooks = Bag(
122 device=as_object('systemimage.device.SystemProperty'),
123 scorer=as_object('systemimage.scores.WeightedScorer'),
124 apply=as_object('systemimage.apply.Reboot'),
125 )
126 self.dbus = Bag(
127 lifetime=as_timedelta('10m'),
128 )
129
130 def _load_file(self, path):
131 parser = SafeConfigParser()
132 str_path = str(path)
133 parser.read(str_path)
134 self.ini_files.append(path)
135 self.service.update(converters=dict(http_port=as_port,
136 https_port=as_port,
90 build_number=int,137 build_number=int,
91 device=device_converter,138 device=as_stripped,
92 ),139 ),
93 **parser['service'])140 **parser['service'])
94 if (self.service.http_port is DISABLED and141 self.system.update(converters=dict(timeout=as_timedelta,
95 self.service.https_port is DISABLED):142 loglevel=as_loglevel,
143 settings_db=expand_path,
144 tempdir=expand_path),
145 **parser['system'])
146 self.gpg.update(**parser['gpg'])
147 self.updater.update(**parser['updater'])
148 self.hooks.update(converters=dict(device=as_object,
149 scorer=as_object,
150 apply=as_object),
151 **parser['hooks'])
152 self.dbus.update(converters=dict(lifetime=as_timedelta),
153 **parser['dbus'])
154
155 def load(self, directory):
156 """Load up the configuration from a config.d directory."""
157 # Look for all the files in the given directory with .ini or .cfg
158 # suffixes. The files must start with a number, and the files are
159 # loaded in numeric order.
160 if self.config_d is not None:
161 raise RuntimeError('Configuration already loaded; use .reload()')
162 self.config_d = directory
163 if not Path(directory).is_dir():
164 raise TypeError(
165 '.load() requires a directory: {}'.format(directory))
166 candidates = []
167 for child in Path(directory).glob('*.ini'):
168 order, _, base = child.stem.partition('_')
169 # XXX 2014-10-03: The logging system isn't initialized when we get
170 # here, so we can't log that these files are being ignored.
171 if len(_) == 0:
172 continue
173 try:
174 serial = int(order)
175 except ValueError:
176 continue
177 candidates.append((serial, child))
178 for serial, path in sorted(candidates):
179 self._load_file(path)
180 self._calculate_http_bases()
181
182 def reload(self):
183 """Reload the configuration directory."""
184 # Reset some cached attributes.
185 directory = self.config_d
186 self.ini_files = []
187 self.config_d = None
188 self._build_number = None
189 # Now load the defaults, then reload the previous config.d directory.
190 self._set_defaults()
191 self.load(directory)
192
193 def _calculate_http_bases(self):
194 if (self.service.http_port is NO_PORT and
195 self.service.https_port is NO_PORT):
96 raise ValueError('Cannot disable both http and https ports')196 raise ValueError('Cannot disable both http and https ports')
97 # Construct the HTTP and HTTPS base urls, which most applications will197 # Construct the HTTP and HTTPS base urls, which most applications will
98 # actually use. We do this in two steps, in order to support198 # actually use. We do this in two steps, in order to support disabling
99 # disabling one or the other (but not both) protocols.199 # one or the other (but not both) protocols.
100 if self.service.http_port == 80:200 if self.service.http_port == 80:
101 http_base = 'http://{}'.format(self.service.base)201 http_base = 'http://{}'.format(self.service.base)
102 elif self.service.http_port is DISABLED:202 elif self.service.http_port is NO_PORT:
103 http_base = None203 http_base = None
104 else:204 else:
105 http_base = 'http://{}:{}'.format(205 http_base = 'http://{}:{}'.format(
@@ -107,7 +207,7 @@
107 # HTTPS.207 # HTTPS.
108 if self.service.https_port == 443:208 if self.service.https_port == 443:
109 https_base = 'https://{}'.format(self.service.base)209 https_base = 'https://{}'.format(self.service.base)
110 elif self.service.https_port is DISABLED:210 elif self.service.https_port is NO_PORT:
111 https_base = None211 https_base = None
112 else:212 else:
113 https_base = 'https://{}:{}'.format(213 https_base = 'https://{}:{}'.format(
@@ -119,45 +219,13 @@
119 if https_base is None:219 if https_base is None:
120 assert http_base is not None220 assert http_base is not None
121 https_base = http_base221 https_base = http_base
122 self.service['http_base'] = http_base222 self.http_base = http_base
123 self.service['https_base'] = https_base223 self.https_base = https_base
124 try:
125 self.system.update(converters=dict(timeout=as_timedelta,
126 build_file=expand_path,
127 loglevel=as_loglevel,
128 settings_db=expand_path,
129 tempdir=expand_path),
130 **parser['system'])
131 except KeyError:
132 # If we're overriding via a channel.ini file, it's okay if the
133 # [system] section is missing. However, the main configuration
134 # ini file must include all sections.
135 if not override:
136 raise
137 # Short-circuit, since we're loading a channel.ini file.
138 self._override = override
139 if override:
140 return
141 self.gpg = Bag(**parser['gpg'])
142 self.updater = Bag(**parser['updater'])
143 self.hooks = Bag(converters=dict(device=as_object,
144 scorer=as_object,
145 reboot=as_object),
146 **parser['hooks'])
147 self.dbus = Bag(converters=dict(lifetime=as_timedelta),
148 **parser['dbus'])
149224
150 @property225 @property
151 def build_number(self):226 def build_number(self):
152 if self._build_number is None:227 if self._build_number is None:
153 if self._override:228 self._build_number = self.service.build_number
154 return self.service.build_number
155 else:
156 try:
157 with open(self.system.build_file, encoding='utf-8') as fp:
158 return int(fp.read().strip())
159 except FileNotFoundError:
160 return 0
161 return self._build_number229 return self._build_number
162230
163 @build_number.setter231 @build_number.setter
@@ -166,24 +234,18 @@
166 raise ValueError(234 raise ValueError(
167 'integer is required, got: {}'.format(type(value).__name__))235 'integer is required, got: {}'.format(type(value).__name__))
168 self._build_number = value236 self._build_number = value
237 self.build_number_override = True
169238
170 @build_number.deleter239 @build_number.deleter
171 def build_number(self):240 def build_number(self):
172 self._build_number = None241 self._build_number = None
173242
174 @property243 @property
175 def build_number_cli(self):
176 return self._build_number
177
178 @property
179 def device(self):244 def device(self):
180 if self._device is None:245 if self._device is None:
181 # Start by looking for a [service]device setting. Use this if it246 # Start by looking for a [service]device setting. Use this if it
182 # exists, otherwise fall back to calling the hook.247 # exists, otherwise fall back to calling the hook.
183 self._device = getattr(self.service, 'device', None)248 self._device = getattr(self.service, 'device', None)
184 # The key could exist in the channel.ini file, but its value could
185 # be empty. That's semantically equivalent to a missing
186 # [service]device setting.
187 if not self._device:249 if not self._device:
188 self._device = self.hooks.device().get_device()250 self._device = self.hooks.device().get_device()
189 return self._device251 return self._device
@@ -203,6 +265,18 @@
203 self._channel = value265 self._channel = value
204266
205 @property267 @property
268 def phase_override(self):
269 return self._phase_override
270
271 @phase_override.setter
272 def phase_override(self, value):
273 self._phase_override = max(0, min(100, int(value)))
274
275 @phase_override.deleter
276 def phase_override(self):
277 self._phase_override = None
278
279 @property
206 def tempdir(self):280 def tempdir(self):
207 if self._tempdir is None:281 if self._tempdir is None:
208 makedirs(self.system.tempdir)282 makedirs(self.system.tempdir)
@@ -211,21 +285,13 @@
211 dir=self.system.tempdir))285 dir=self.system.tempdir))
212 return self._tempdir286 return self._tempdir
213287
214288 @property
215# Define the global configuration object. Normal use can be as simple as:289 def user_agent(self):
216#290 return USER_AGENT.format(self)
217# from systemimage.config import config291
218# build_file = config.system.build_file292
219#293# Define the global configuration object. We use a proxy here so that
220# In the test suite though, the actual configuration object can be easily294# post-object creation loading will work.
221# patched by doing something like this:
222#
223# test_config = Configuration(...)
224# with unittest.mock.patch('config._config', test_config):
225# run_test()
226#
227# and now every module which does the first code example will get build_file
228# from the mocked Configuration instance.
229295
230_config = Configuration()296_config = Configuration()
231297
232298
=== added file 'systemimage/curl.py'
--- systemimage/curl.py 1970-01-01 00:00:00 +0000
+++ systemimage/curl.py 2015-05-20 14:55:53 +0000
@@ -0,0 +1,275 @@
1# Copyright (C) 2014-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>
3
4# This program is free software: you can redistribute it and/or modify
5# it under the terms of the GNU General Public License as published by
6# the Free Software Foundation; version 3 of the License.
7#
8# This program is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU General Public License for more details.
12#
13# You should have received a copy of the GNU General Public License
14# along with this program. If not, see <http://www.gnu.org/licenses/>.
15
16"""Download files via PyCURL."""
17
18__all__ = [
19 'CurlDownloadManager',
20 ]
21
22
23import pycurl
24import hashlib
25import logging
26
27from contextlib import ExitStack
28from gi.repository import GLib
29from systemimage.config import config
30from systemimage.download import Canceled, DownloadManagerBase
31
32log = logging.getLogger('systemimage')
33
34
35# Some cURL defaults. XXX pull these out of the configuration file.
36CONNECTION_TIMEOUT = 120 # seconds
37LOW_SPEED_LIMIT = 10
38LOW_SPEED_TIME = 120 # seconds
39MAX_REDIRECTS = 5
40MAX_TOTAL_CONNECTIONS = 4
41SELECT_TIMEOUT = 0.05 # 20fps
42
43
44def _curl_debug(debug_type, debug_msg): # pragma: no cover
45 from systemimage.testing.helpers import debug
46 with debug(end='') as ddlog:
47 ddlog('PYCURL:', debug_type, debug_msg)
48
49
50def make_testable(c):
51 # The test suite needs to make the PyCURL object accept the testing
52 # server's self signed certificate. It will mock this function.
53 pass
54
55
56class SingleDownload:
57 def __init__(self, record):
58 self.url, self.destination, self.expected_checksum = record
59 self._checksum = None
60 self._fp = None
61 self._resources = ExitStack()
62
63 def make_handle(self, *, HEAD):
64 # If we're doing GET, record some more information.
65 if not HEAD:
66 self._checksum = hashlib.sha256()
67 # Create the basic PyCURL object.
68 c = pycurl.Curl()
69 # Set the common options.
70 c.setopt(pycurl.URL, self.url)
71 c.setopt(pycurl.USERAGENT, config.user_agent)
72 # If we're doing a HEAD, then we don't want the body of the
73 # file. Otherwise, set things up to write the body data to the
74 # destination file.
75 if HEAD:
76 c.setopt(pycurl.NOBODY, 1)
77 else:
78 c.setopt(pycurl.WRITEDATA, self)
79 self._fp = self._resources.enter_context(
80 open(self.destination, 'wb'))
81 # Set some limits. XXX Pull these out of the configuration files.
82 c.setopt(pycurl.FOLLOWLOCATION, 1)
83 c.setopt(pycurl.MAXREDIRS, MAX_REDIRECTS)
84 c.setopt(pycurl.CONNECTTIMEOUT, CONNECTION_TIMEOUT)
85 # If the average transfer speed is below 10 bytes per second for 2
86 # minutes, libcurl will consider the connection too slow and abort.
87 ## c.setopt(pycurl.LOW_SPEED_LIMIT, LOW_SPEED_LIMIT)
88 ## c.setopt(pycurl.LOW_SPEED_TIME, LOW_SPEED_TIME)
89 # Fail on error codes >= 400.
90 c.setopt(pycurl.FAILONERROR, 1)
91 # Switch off the libcurl progress meters. The multi that uses
92 # this handle will set the transfer info function.
93 c.setopt(pycurl.NOPROGRESS, 1)
94 # ssl: no need to set SSL_VERIFYPEER, SSL_VERIFYHOST, CAINFO
95 # they all use sensible defaults
96 #
97 # Enable debugging.
98 self._make_debuggable(c)
99 # For the test suite.
100 make_testable(c)
101 return c
102
103 def _make_debuggable(self, c):
104 """Add some additional debugging options."""
105 ## c.setopt(pycurl.VERBOSE, 1)
106 ## c.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
107 pass
108
109 def write(self, data):
110 """Update the checksum and write the data out to the file."""
111 self._checksum.update(data)
112 self._fp.write(data)
113 # Returning None implies that all bytes were written
114 # successfully, so it's better to be explicit.
115 return None
116
117 def close(self):
118 self._resources.close()
119
120 @property
121 def checksum(self):
122 # If no checksum was expected, pretend none was gotten. This
123 # makes the verification step below a wee bit simpler.
124 if self.expected_checksum == '':
125 return ''
126 return self._checksum.hexdigest()
127
128
129class CurlDownloadManager(DownloadManagerBase):
130 """The PyCURL based download manager."""
131
132 def __init__(self, callback=None):
133 super().__init__()
134 if callback is not None:
135 self.callbacks.append(callback)
136 self._pausables = []
137 self._paused = False
138
139 def _get_files(self, records, pausable):
140 # Start by doing a HEAD on all the URLs so that we can get the total
141 # target download size in bytes, at least as best as is possible.
142 with ExitStack() as resources:
143 handles = []
144 multi = pycurl.CurlMulti()
145 multi.setopt(
146 pycurl.M_MAX_TOTAL_CONNECTIONS, MAX_TOTAL_CONNECTIONS)
147 for record in records:
148 download = SingleDownload(record)
149 resources.callback(download.close)
150 handle = download.make_handle(HEAD=True)
151 handles.append(handle)
152 multi.add_handle(handle)
153 # .add_handle() does not bump the reference count, so we
154 # need to keep the PyCURL object alive for the duration
155 # of this download.
156 resources.callback(multi.remove_handle, handle)
157 self._perform(multi, handles)
158 self.total = sum(
159 handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
160 for handle in handles)
161 # Now do a GET on all the URLs. This will write the data to the
162 # destination file and collect the checksums.
163 with ExitStack() as resources:
164 resources.callback(setattr, self, '_handles', None)
165 downloads = []
166 multi = pycurl.CurlMulti()
167 multi.setopt(
168 pycurl.M_MAX_TOTAL_CONNECTIONS, MAX_TOTAL_CONNECTIONS)
169 for record in records:
170 download = SingleDownload(record)
171 downloads.append(download)
172 resources.callback(download.close)
173 handle = download.make_handle(HEAD=False)
174 self._pausables.append(handle)
175 multi.add_handle(handle)
176 # .add_handle() does not bump the reference count, so we
177 # need to keep the PyCURL object alive for the duration
178 # of this download.
179 resources.callback(multi.remove_handle, handle)
180 self._perform(multi, self._pausables)
181 # Verify internally calculated checksums. The API requires
182 # a FileNotFoundError to be raised when they don't match.
183 # Since it doesn't matter which one fails, log them all and
184 # raise the first one.
185 first_mismatch = None
186 for download in downloads:
187 if download.checksum != download.expected_checksum:
188 log.error('Checksum mismatch. got:{} != exp:{}: {}',
189 download.checksum, download.expected_checksum,
190 download.destination)
191 if first_mismatch is None:
192 first_mismatch = download
193 if first_mismatch is not None:
194 # For backward compatibility with ubuntu-download_manager.
195 raise FileNotFoundError('HASH ERROR: {}'.format(
196 first_mismatch.destination))
197 self._pausables = []
198
199 def _do_once(self, multi, handles):
200 status, active_count = multi.perform()
201 if status == pycurl.E_CALL_MULTI_PERFORM:
202 # Call .perform() again before calling select.
203 return True
204 elif status != pycurl.E_OK:
205 # An error occurred in the multi, so be done with the
206 # whole thing. We can't get a description string out of
207 # PyCURL though. Just raise one of the urls.
208 log.error('CurlMulti() error: {}', status)
209 raise FileNotFoundError(handles[0].getinfo(pycurl.EFFECTIVE_URL))
210 # The multi is okay, but it's possible there are errors pending on
211 # the individual downloads; check those now.
212 queued_count, ok_list, error_list = multi.info_read()
213 if len(error_list) > 0:
214 # It helps to have at least one URL in the FileNotFoundError.
215 first_url = None
216 log.error('Curl() errors encountered:')
217 for c, code, message in error_list:
218 url = c.getinfo(pycurl.EFFECTIVE_URL)
219 if first_url is None:
220 first_url = url
221 log.error(' {} ({}): {}', message, code, url)
222 raise FileNotFoundError('{}: {}'.format(message, first_url))
223 # For compatibility with .io_add_watch(), we return False if we want
224 # to stop the callbacks, and True if we want to call back here again.
225 return active_count > 0
226
227 def _perform(self, multi, handles):
228 # While we're performing the cURL downloads, we need to periodically
229 # process D-Bus events, otherwise we won't be able to cancel downloads
230 # or handle other interruptive events. To do this, we grab the GLib
231 # main loop context and then ask it to do an iteration over its events
232 # once in a while. It turns out that even if we're not running a D-Bus
233 # main loop (i.e. during the in-process tests) periodically dispatching
234 # into GLib doesn't hurt, so just do it unconditionally.
235 self.received = 0
236 context = GLib.main_context_default()
237 while True:
238 # Do the progress callback, but only if the current received size
239 # is different than the last one. Don't worry about in which
240 # direction it's different.
241 received = int(
242 sum(c.getinfo(pycurl.SIZE_DOWNLOAD) for c in handles))
243 if received != self.received:
244 self._do_callback()
245 self.received = received
246 if not self._do_once(multi, handles):
247 break
248 multi.select(SELECT_TIMEOUT)
249 # Let D-Bus events get dispatched, but only block if downloads are
250 # paused.
251 while context.iteration(may_block=self._paused):
252 pass
253 if self._queued_cancel:
254 raise Canceled
255 # One last callback, unconditionally.
256 self.received = int(
257 sum(c.getinfo(pycurl.SIZE_DOWNLOAD) for c in handles))
258 self._do_callback()
259
260 def pause(self):
261 for c in self._pausables:
262 c.pause(pycurl.PAUSE_ALL)
263 self._paused = True
264 # 2014-10-20 BAW: We could plumb through the `service` object from
265 # service.py (the main entry point for system-image-dbus, but that's
266 # actually a bit of a pain, so do the expedient thing and grab the
267 # interface here.
268 percentage = (int(self.received / self.total * 100.0)
269 if self.total > 0 else 0)
270 config.dbus_service.UpdatePaused(percentage)
271
272 def resume(self):
273 self._paused = False
274 for c in self._pausables:
275 c.pause(pycurl.PAUSE_CONT)
0276
=== removed file 'systemimage/data/client.ini'
--- systemimage/data/client.ini 2014-01-30 15:41:03 +0000
+++ systemimage/data/client.ini 1970-01-01 00:00:00 +0000
@@ -1,35 +0,0 @@
1# Default and example .ini configuration file.
2# Edit this and put it in /etc/system-image/client.ini
3
4[service]
5base: system-image.ubuntu.com
6http_port: 80
7https_port: 443
8channel: daily
9build_number: 0
10
11[system]
12timeout: 1h
13build_file: /etc/ubuntu-build
14tempdir: /tmp
15logfile: /var/log/system-image/client.log
16loglevel: info
17settings_db: /var/lib/system-image/settings.db
18
19[gpg]
20archive_master: /etc/system-image/archive-master.tar.xz
21image_master: /var/lib/system-image/keyrings/image-master.tar.xz
22image_signing: /var/lib/system-image/keyrings/image-signing.tar.xz
23device_signing: /var/lib/system-image/keyrings/device-signing.tar.xz
24
25[updater]
26cache_partition: /android/cache/recovery
27data_partition: /var/lib/system-image
28
29[hooks]
30device: systemimage.device.SystemProperty
31scorer: systemimage.scores.WeightedScorer
32reboot: systemimage.reboot.Reboot
33
34[dbus]
35lifetime: 10m
360
=== modified file 'systemimage/dbus.py'
--- systemimage/dbus.py 2014-09-26 14:36:34 +0000
+++ systemimage/dbus.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -94,14 +94,14 @@
9494
95 def __init__(self, bus, object_path, loop):95 def __init__(self, bus, object_path, loop):
96 super().__init__(bus, object_path)96 super().__init__(bus, object_path)
97 self._loop = loop97 self.loop = loop
98 self._api = Mediator(self._progress_callback)98 self._api = Mediator(self._progress_callback)
99 log.info('Mediator created {}', self._api)99 log.info('Mediator created {}', self._api)
100 self._checking = Lock()100 self._checking = Lock()
101 self._downloading = Lock()
101 self._update = None102 self._update = None
102 self._downloading = False
103 self._paused = False103 self._paused = False
104 self._rebootable = False104 self._applicable = False
105 self._failure_count = 0105 self._failure_count = 0
106 self._last_error = ''106 self._last_error = ''
107107
@@ -110,8 +110,15 @@
110 # Asynchronous method call.110 # Asynchronous method call.
111 log.info('Enter _check_for_update()')111 log.info('Enter _check_for_update()')
112 self._update = self._api.check_for_update()112 self._update = self._api.check_for_update()
113 log.info('_check_for_update(): checking lock releasing')
114 try:
115 self._checking.release()
116 except RuntimeError:
117 log.info('_check_for_update(): checking lock already released')
118 else:
119 log.info('_check_for_update(): checking lock released')
113 # Do we have an update and can we auto-download it?120 # Do we have an update and can we auto-download it?
114 downloading = False121 delayed_download = False
115 if self._update.is_available:122 if self._update.is_available:
116 settings = Settings()123 settings = Settings()
117 auto = settings.get('auto_download')124 auto = settings.get('auto_download')
@@ -119,14 +126,19 @@
119 if auto in ('1', '2'):126 if auto in ('1', '2'):
120 # XXX When we have access to the download service, we can127 # XXX When we have access to the download service, we can
121 # check if we're on the wifi (auto == '1').128 # check if we're on the wifi (auto == '1').
129 delayed_download = True
122 GLib.timeout_add(50, self._download)130 GLib.timeout_add(50, self._download)
123 downloading = True131 # We have a timing issue. We can't lock the downloading lock here,
132 # otherwise when _download() starts running in ~50ms it will think a
133 # download is already in progress. But we want to send the UAS signal
134 # here and now, *and* indicate whether the download is about to happen.
135 # So just lie for now since in ~50ms the download will begin.
124 self.UpdateAvailableStatus(136 self.UpdateAvailableStatus(
125 self._update.is_available,137 self._update.is_available,
126 downloading,138 delayed_download,
127 self._update.version,139 self._update.version,
128 self._update.size,140 self._update.size,
129 self._update.last_update_date,141 last_update_date(),
130 self._update.error)142 self._update.error)
131 # Stop GLib from calling this method again.143 # Stop GLib from calling this method again.
132 return False144 return False
@@ -148,24 +160,24 @@
148 completes. The argument to that signal is a boolean indicating160 completes. The argument to that signal is a boolean indicating
149 whether the update is available or not.161 whether the update is available or not.
150 """162 """
151 self._loop.keepalive()163 self.loop.keepalive()
152 # Check-and-acquire the lock.164 # Check-and-acquire the lock.
153 log.info('test and acquire checking lock')165 log.info('CheckForUpdate(): checking lock test and acquire')
154 if not self._checking.acquire(blocking=False):166 if not self._checking.acquire(blocking=False):
167 log.info('CheckForUpdate(): checking lock not acquired')
155 # Check is already in progress, so there's nothing more to do. If168 # Check is already in progress, so there's nothing more to do. If
156 # there's status available (i.e. we are in the auto-downloading169 # there's status available (i.e. we are in the auto-downloading
157 # phase of the last CFU), then send the status.170 # phase of the last CFU), then send the status.
158 if self._update is not None:171 if self._update is not None:
159 self.UpdateAvailableStatus(172 self.UpdateAvailableStatus(
160 self._update.is_available,173 self._update.is_available,
161 self._downloading,174 self._downloading.locked(),
162 self._update.version,175 self._update.version,
163 self._update.size,176 self._update.size,
164 self._update.last_update_date,177 last_update_date(),
165 "")178 "")
166 log.info('checking lock not acquired')
167 return179 return
168 log.info('checking lock acquired')180 log.info('CheckForUpdate(): checking lock acquired')
169 # We've now acquired the lock. Reset any failure or in-progress181 # We've now acquired the lock. Reset any failure or in-progress
170 # state. Get a new mediator to reset any of its state.182 # state. Get a new mediator to reset any of its state.
171 self._api = Mediator(self._progress_callback)183 self._api = Mediator(self._progress_callback)
@@ -176,7 +188,7 @@
176 # this method can return immediately.188 # this method can return immediately.
177 GLib.timeout_add(50, self._check_for_update)189 GLib.timeout_add(50, self._check_for_update)
178190
179 @log_and_exit191 #@log_and_exit
180 def _progress_callback(self, received, total):192 def _progress_callback(self, received, total):
181 # Plumb the progress through our own D-Bus API. Our API is defined as193 # Plumb the progress through our own D-Bus API. Our API is defined as
182 # signalling a percentage and an eta. We can calculate the percentage194 # signalling a percentage and an eta. We can calculate the percentage
@@ -187,12 +199,12 @@
187199
188 @log_and_exit200 @log_and_exit
189 def _download(self):201 def _download(self):
190 if self._downloading and self._paused:202 if self._downloading.locked() and self._paused:
191 self._api.resume()203 self._api.resume()
192 self._paused = False204 self._paused = False
193 log.info('Download previously paused')205 log.info('Download previously paused')
194 return206 return
195 if (self._downloading # Already in progress.207 if (self._downloading.locked() # Already in progress.
196 or self._update is None # Not yet checked.208 or self._update is None # Not yet checked.
197 or not self._update.is_available # No update available.209 or not self._update.is_available # No update available.
198 ):210 ):
@@ -204,47 +216,33 @@
204 log.info('Update failures: {}; last error: {}',216 log.info('Update failures: {}; last error: {}',
205 self._failure_count, self._last_error)217 self._failure_count, self._last_error)
206 return218 return
207 self._downloading = True219 log.info('_download(): downloading lock entering critical section')
208 log.info('Update is downloading')220 with self._downloading:
209 try:221 log.info('Update is downloading')
210 # Always start by sending a UpdateProgress(0, 0). This is222 try:
211 # enough to get the u/i's attention.223 # Always start by sending a UpdateProgress(0, 0). This is
212 self.UpdateProgress(0, 0)224 # enough to get the u/i's attention.
213 self._api.download()225 self.UpdateProgress(0, 0)
214 except Exception:226 self._api.download()
215 log.exception('Download failed')227 except Exception:
216 self._failure_count += 1228 log.exception('Download failed')
217 # Set the last error string to the exception's class name.229 self._failure_count += 1
218 exception, value = sys.exc_info()[:2]230 # Set the last error string to the exception's class name.
219 # if there's no meaningful value, omit it.231 exception, value = sys.exc_info()[:2]
220 value_str = str(value)232 # if there's no meaningful value, omit it.
221 name = exception.__name__233 value_str = str(value)
222 self._last_error = ('{}'.format(name)234 name = exception.__name__
223 if len(value_str) == 0235 self._last_error = ('{}'.format(name)
224 else '{}: {}'.format(name, value))236 if len(value_str) == 0
225 self.UpdateFailed(self._failure_count, self._last_error)237 else '{}: {}'.format(name, value))
226 else:238 self.UpdateFailed(self._failure_count, self._last_error)
227 log.info('Update downloaded')239 else:
228 self.UpdateDownloaded()240 log.info('Update downloaded')
229 self._failure_count = 0241 self.UpdateDownloaded()
230 self._last_error = ''242 self._failure_count = 0
231 self._rebootable = True243 self._last_error = ''
232 self._downloading = False244 self._applicable = True
233 log.info('releasing checking lock from _download()')245 log.info('_download(): downloading lock finished critical section')
234 try:
235 self._checking.release()
236 except RuntimeError:
237 # 2014-09-11 BAW: We don't own the lock. There are several reasons
238 # why this can happen including: 1) the client canceled the
239 # download while it was in progress, and CancelUpdate() already
240 # released the lock; 2) the client called DownloadUpdate() without
241 # first calling CheckForUpdate(); 3) the client called DU()
242 # multiple times in a row but the update was already downloaded and
243 # all the file signatures have been verified. I can't think of
244 # reason why we shouldn't just ignore the double release, so
245 # that's what we do. See LP: #1365646.
246 pass
247 log.info('released checking lock from _download()')
248 # Stop GLib from calling this method again.246 # Stop GLib from calling this method again.
249 return False247 return False
250248
@@ -257,15 +255,15 @@
257 """255 """
258 # Arrange for the update to happen in a little while, so that this256 # Arrange for the update to happen in a little while, so that this
259 # method can return immediately.257 # method can return immediately.
260 self._loop.keepalive()258 self.loop.keepalive()
261 GLib.timeout_add(50, self._download)259 GLib.timeout_add(50, self._download)
262260
263 @log_and_exit261 @log_and_exit
264 @method('com.canonical.SystemImage', out_signature='s')262 @method('com.canonical.SystemImage', out_signature='s')
265 def PauseDownload(self):263 def PauseDownload(self):
266 """Pause a downloading update."""264 """Pause a downloading update."""
267 self._loop.keepalive()265 self.loop.keepalive()
268 if self._downloading:266 if self._downloading.locked():
269 self._api.pause()267 self._api.pause()
270 self._paused = True268 self._paused = True
271 error_message = ''269 error_message = ''
@@ -277,40 +275,34 @@
277 @method('com.canonical.SystemImage', out_signature='s')275 @method('com.canonical.SystemImage', out_signature='s')
278 def CancelUpdate(self):276 def CancelUpdate(self):
279 """Cancel a download."""277 """Cancel a download."""
280 self._loop.keepalive()278 self.loop.keepalive()
281 # During the download, this will cause an UpdateFailed signal to be279 # During the download, this will cause an UpdateFailed signal to be
282 # issued, as part of the exception handling in _download(). If we're280 # issued, as part of the exception handling in _download(). If we're
283 # not downloading, then no signal need be sent. There's no need to281 # not downloading, then no signal need be sent. There's no need to
284 # send *another* signal when downloading, because we never will be282 # send *another* signal when downloading, because we never will be
285 # downloading by the time we get past this next call.283 # downloading by the time we get past this next call.
286 self._api.cancel()284 self._api.cancel()
287 # If we're holding the checking lock, release it.
288 try:
289 log.info('releasing checking lock from CancelUpdate()')
290 self._checking.release()
291 log.info('released checking lock from CancelUpdate()')
292 except RuntimeError:
293 # We're not holding the lock.
294 pass
295 # XXX 2013-08-22: If we can't cancel the current download, return the285 # XXX 2013-08-22: If we can't cancel the current download, return the
296 # reason in this string.286 # reason in this string.
297 return ''287 return ''
298288
299 @log_and_exit289 @log_and_exit
300 def _apply_update(self):290 def _apply_update(self):
301 self._loop.keepalive()291 self.loop.keepalive()
302 if not self._rebootable:292 if not self._applicable:
303 command_file = os.path.join(293 command_file = os.path.join(
304 config.updater.cache_partition, 'ubuntu_command')294 config.updater.cache_partition, 'ubuntu_command')
305 if not os.path.exists(command_file):295 if not os.path.exists(command_file):
306 # Not enough has been downloaded to allow for a reboot.296 # Not enough has been downloaded to allow for the update to be
307 self.Rebooting(False)297 # applied.
298 self.Applied(False)
308 return299 return
309 self._api.reboot()300 self._api.apply()
310 # This code may or may not run. We're racing against the system301 # This code may or may not run. On devices for which applying the
311 # reboot procedure.302 # update requires a system reboot, we're racing against that reboot
312 self._rebootable = False303 # procedure.
313 self.Rebooting(True)304 self._applicable = False
305 self.Applied(True)
314306
315 @log_and_exit307 @log_and_exit
316 @method('com.canonical.SystemImage')308 @method('com.canonical.SystemImage')
@@ -322,7 +314,7 @@
322 @log_and_exit314 @log_and_exit
323 @method('com.canonical.SystemImage', out_signature='isssa{ss}')315 @method('com.canonical.SystemImage', out_signature='isssa{ss}')
324 def Info(self):316 def Info(self):
325 self._loop.keepalive()317 self.loop.keepalive()
326 return (config.build_number,318 return (config.build_number,
327 config.device,319 config.device,
328 config.channel,320 config.channel,
@@ -332,23 +324,27 @@
332 @log_and_exit324 @log_and_exit
333 @method('com.canonical.SystemImage', out_signature='a{ss}')325 @method('com.canonical.SystemImage', out_signature='a{ss}')
334 def Information(self):326 def Information(self):
335 self._loop.keepalive()327 self.loop.keepalive()
336 settings = Settings()328 settings = Settings()
337 current_build_number = str(config.build_number)329 current_build_number = str(config.build_number)
330 version_detail = getattr(config.service, 'version_detail', '')
338 response = dict(331 response = dict(
339 current_build_number=current_build_number,332 current_build_number=current_build_number,
340 device_name=config.device,333 device_name=config.device,
341 channel_name=config.channel,334 channel_name=config.channel,
342 last_update_date=last_update_date(),335 last_update_date=last_update_date(),
343 version_detail=getattr(config.service, 'version_detail', ''),336 version_detail=version_detail,
344 last_check_date=settings.get('last_check_date'),337 last_check_date=settings.get('last_check_date'),
345 )338 )
346 if self._update is None:339 if self._update is None:
347 response['target_build_number'] = '-1'340 response['target_build_number'] = '-1'
341 response['target_version_detail'] = ''
348 elif not self._update.is_available:342 elif not self._update.is_available:
349 response['target_build_number'] = current_build_number343 response['target_build_number'] = current_build_number
344 response['target_version_detail'] = version_detail
350 else:345 else:
351 response['target_build_number'] = str(self._update.version)346 response['target_build_number'] = str(self._update.version)
347 response['target_version_detail'] = self._update.version_detail
352 return response348 return response
353349
354 @log_and_exit350 @log_and_exit
@@ -359,7 +355,7 @@
359 Some values are special, e.g. min_battery and auto_downloads.355 Some values are special, e.g. min_battery and auto_downloads.
360 Implement these special semantics here.356 Implement these special semantics here.
361 """357 """
362 self._loop.keepalive()358 self.loop.keepalive()
363 if key == 'min_battery':359 if key == 'min_battery':
364 try:360 try:
365 as_int = int(value)361 as_int = int(value)
@@ -385,22 +381,24 @@
385 @method('com.canonical.SystemImage', in_signature='s', out_signature='s')381 @method('com.canonical.SystemImage', in_signature='s', out_signature='s')
386 def GetSetting(self, key):382 def GetSetting(self, key):
387 """Get a setting."""383 """Get a setting."""
388 self._loop.keepalive()384 self.loop.keepalive()
389 return Settings().get(key)385 return Settings().get(key)
390386
391 @log_and_exit387 @log_and_exit
392 @method('com.canonical.SystemImage')388 @method('com.canonical.SystemImage')
393 def FactoryReset(self):389 def FactoryReset(self):
394 self._api.factory_reset()390 self._api.factory_reset()
395 # This code may or may not run. We're racing against the system391
396 # reboot procedure.392 @log_and_exit
397 self.Rebooting(True)393 @method('com.canonical.SystemImage')
394 def ProductionReset(self):
395 self._api.production_reset()
398396
399 @log_and_exit397 @log_and_exit
400 @method('com.canonical.SystemImage')398 @method('com.canonical.SystemImage')
401 def Exit(self):399 def Exit(self):
402 """Quit the daemon immediately."""400 """Quit the daemon immediately."""
403 self._loop.quit()401 self.loop.quit()
404402
405 @log_and_exit403 @log_and_exit
406 @signal('com.canonical.SystemImage', signature='bbsiss')404 @signal('com.canonical.SystemImage', signature='bbsiss')
@@ -416,21 +414,21 @@
416 log.debug('EMIT UpdateAvailableStatus({}, {}, {}, {}, {}, {})',414 log.debug('EMIT UpdateAvailableStatus({}, {}, {}, {}, {}, {})',
417 is_available, downloading, available_version, update_size,415 is_available, downloading, available_version, update_size,
418 last_update_date, repr(error_reason))416 last_update_date, repr(error_reason))
419 self._loop.keepalive()417 self.loop.keepalive()
420418
421 @log_and_exit419 #@log_and_exit
422 @signal('com.canonical.SystemImage', signature='id')420 @signal('com.canonical.SystemImage', signature='id')
423 def UpdateProgress(self, percentage, eta):421 def UpdateProgress(self, percentage, eta):
424 """Download progress."""422 """Download progress."""
425 log.debug('EMIT UpdateProgress({}, {})', percentage, eta)423 log.debug('EMIT UpdateProgress({}, {})', percentage, eta)
426 self._loop.keepalive()424 self.loop.keepalive()
427425
428 @log_and_exit426 @log_and_exit
429 @signal('com.canonical.SystemImage')427 @signal('com.canonical.SystemImage')
430 def UpdateDownloaded(self):428 def UpdateDownloaded(self):
431 """The update has been successfully downloaded."""429 """The update has been successfully downloaded."""
432 log.debug('EMIT UpdateDownloaded()')430 log.debug('EMIT UpdateDownloaded()')
433 self._loop.keepalive()431 self.loop.keepalive()
434432
435 @log_and_exit433 @log_and_exit
436 @signal('com.canonical.SystemImage', signature='is')434 @signal('com.canonical.SystemImage', signature='is')
@@ -438,21 +436,28 @@
438 """The update failed for some reason."""436 """The update failed for some reason."""
439 log.debug('EMIT UpdateFailed({}, {})',437 log.debug('EMIT UpdateFailed({}, {})',
440 consecutive_failure_count, repr(last_reason))438 consecutive_failure_count, repr(last_reason))
441 self._loop.keepalive()439 self.loop.keepalive()
442440
443 @log_and_exit441 @log_and_exit
444 @signal('com.canonical.SystemImage', signature='i')442 @signal('com.canonical.SystemImage', signature='i')
445 def UpdatePaused(self, percentage):443 def UpdatePaused(self, percentage):
446 """The download got paused."""444 """The download got paused."""
447 log.debug('EMIT UpdatePaused({})', percentage)445 log.debug('EMIT UpdatePaused({})', percentage)
448 self._loop.keepalive()446 self.loop.keepalive()
449447
450 @log_and_exit448 @log_and_exit
451 @signal('com.canonical.SystemImage', signature='ss')449 @signal('com.canonical.SystemImage', signature='ss')
452 def SettingChanged(self, key, new_value):450 def SettingChanged(self, key, new_value):
453 """A setting value has change."""451 """A setting value has change."""
454 log.debug('EMIT SettingChanged({}, {})', repr(key), repr(new_value))452 log.debug('EMIT SettingChanged({}, {})', repr(key), repr(new_value))
455 self._loop.keepalive()453 self.loop.keepalive()
454
455 @log_and_exit
456 @signal('com.canonical.SystemImage', signature='b')
457 def Applied(self, status):
458 """The update has been applied."""
459 log.debug('EMIT Applied({})', status)
460 self.loop.keepalive()
456461
457 @log_and_exit462 @log_and_exit
458 @signal('com.canonical.SystemImage', signature='b')463 @signal('com.canonical.SystemImage', signature='b')
459464
=== modified file 'systemimage/device.py'
--- systemimage/device.py 2014-09-17 13:41:31 +0000
+++ systemimage/device.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
55
=== modified file 'systemimage/docs/conf.py'
--- systemimage/docs/conf.py 2014-02-20 23:03:24 +0000
+++ systemimage/docs/conf.py 2015-05-20 14:55:53 +0000
@@ -41,7 +41,7 @@
4141
42# General information about the project.42# General information about the project.
43project = u'Image Update Resolver'43project = u'Image Update Resolver'
44copyright = u'2013-2014, Canonical Ltd.'44copyright = u'2013-2015, Canonical Ltd.'
4545
46# The version info for the project you're documenting, acts as replacement for46# The version info for the project you're documenting, acts as replacement for
47# |version| and |release|, also used in various other places throughout the47# |version| and |release|, also used in various other places throughout the
4848
=== modified file 'systemimage/download.py'
--- systemimage/download.py 2014-09-17 13:41:31 +0000
+++ systemimage/download.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -17,9 +17,9 @@
1717
18__all__ = [18__all__ = [
19 'Canceled',19 'Canceled',
20 'DBusDownloadManager',
21 'DuplicateDestinationError',20 'DuplicateDestinationError',
22 'Record',21 'Record',
22 'get_download_manager',
23 ]23 ]
2424
2525
@@ -28,39 +28,18 @@
28import logging28import logging
2929
30from collections import namedtuple30from collections import namedtuple
31from contextlib import suppress
32from io import StringIO31from io import StringIO
33from pprint import pformat32from pprint import pformat
34from systemimage.config import config33
35from systemimage.reactor import Reactor34try:
36from systemimage.settings import Settings35 import pycurl
3736except ImportError: # pragma: no cover
38# The systemimage.testing module will not be available on installed systems37 pycurl = None
39# unless the system-image-dev binary package is installed, which is not usually
40# the case. Disable _print() debugging in that case.
41def _print(*args, **kws):
42 with suppress(ImportError):
43 # We must import this here to avoid circular imports.
44 from systemimage.testing.helpers import debug
45 with debug(check_flag=True) as ddlog:
46 ddlog(*args, **kws)
47
48
49# Parameterized for testing purposes.
50DOWNLOADER_INTERFACE = 'com.canonical.applications.Downloader'
51MANAGER_INTERFACE = 'com.canonical.applications.DownloadManager'
52OBJECT_NAME = 'com.canonical.applications.Downloader'
53OBJECT_INTERFACE = 'com.canonical.applications.GroupDownload'
54USER_AGENT = 'Ubuntu System Image Upgrade Client; Build {}'
5538
5639
57log = logging.getLogger('systemimage')40log = logging.getLogger('systemimage')
5841
5942
60def _headers():
61 return {'User-Agent': USER_AGENT.format(config.build_number)}
62
63
64class Canceled(Exception):43class Canceled(Exception):
65 """Raised when the download was canceled."""44 """Raised when the download was canceled."""
6645
@@ -89,78 +68,10 @@
89 url=url, destination=destination, checksum=checksum)68 url=url, destination=destination, checksum=checksum)
9069
9170
92class DownloadReactor(Reactor):71class DownloadManagerBase:
93 def __init__(self, bus, callback=None, pausable=False):72 """Base class for all download managers."""
94 super().__init__(bus)73
95 self._callback = callback74 def __init__(self):
96 self._pausable = pausable
97 self.error = None
98 self.canceled = False
99 self.received = 0
100 self.total = 0
101 self.react_to('canceled')
102 self.react_to('error')
103 self.react_to('finished')
104 self.react_to('paused')
105 self.react_to('progress')
106 self.react_to('resumed')
107 self.react_to('started')
108
109 def _do_started(self, signal, path, started):
110 _print('STARTED:', started)
111
112 def _do_finished(self, signal, path, local_paths):
113 _print('FINISHED:', local_paths)
114 self.quit()
115
116 def _do_error(self, signal, path, error_message):
117 _print('ERROR:', error_message)
118 log.error(error_message)
119 self.error = error_message
120 self.quit()
121
122 def _do_progress(self, signal, path, received, total):
123 self.received = received
124 self.total = total
125 _print('PROGRESS:', received, total)
126 if self._callback is not None:
127 # Be defensive, so yes, use a bare except. If an exception occurs
128 # in the callback, log it, but continue onward.
129 try:
130 self._callback(received, total)
131 except:
132 log.exception('Exception in progress callback')
133
134 def _do_canceled(self, signal, path, canceled):
135 # Why would we get this signal if it *wasn't* canceled? Anyway,
136 # this'll be a D-Bus data type so converted it to a vanilla Python
137 # boolean.
138 _print('CANCELED:', canceled)
139 self.canceled = bool(canceled)
140 self.quit()
141
142 def _do_paused(self, signal, path, paused):
143 _print('PAUSE:', paused, self._pausable)
144 send_paused = self._pausable and config.dbus_service is not None
145 if send_paused: # pragma: no branch
146 # We could plumb through the `service` object from service.py (the
147 # main entry point for system-image-dbus, but that's actually a
148 # bit of a pain, so do the expedient thing and grab the interface
149 # here.
150 percentage = (int(self.received / self.total * 100.0)
151 if self.total > 0 else 0)
152 config.dbus_service.UpdatePaused(percentage)
153
154 def _do_resumed(self, signal, path, resumed):
155 _print('RESUME:', resumed)
156 # There currently is no UpdateResumed() signal.
157
158 def _default(self, *args, **kws):
159 _print('SIGNAL:', args, kws) # pragma: no cover
160
161
162class DBusDownloadManager:
163 def __init__(self, callback=None):
164 """75 """
165 :param callback: If given, a function that is called every so often76 :param callback: If given, a function that is called every so often
166 during downloading.77 during downloading.
@@ -168,12 +79,79 @@
168 of bytes received so far, and the total amount of bytes to be79 of bytes received so far, and the total amount of bytes to be
169 downloaded.80 downloaded.
170 """81 """
171 self._iface = None82 # This is a list of functions that are called every so often during
83 # downloading. Functions in this list take two arguments, the number
84 # of bytes received so far, and the total amount of bytes to be
85 # downloaded.
86 self.callbacks = []
87 self.total = 0
88 self.received = 0
172 self._queued_cancel = False89 self._queued_cancel = False
173 self.callback = callback
17490
175 def __repr__(self): # pragma: no cover91 def __repr__(self): # pragma: no cover
176 return '<DBusDownloadManager at 0x{:x}>'.format(id(self))92 return '<{} at 0x{:x}>'.format(self.__class__.__name__, id(self))
93
94 def _get_download_records(self, downloads):
95 """Convert the downloads items to download records."""
96 records = [item if isinstance(item, _RecordType) else Record(*item)
97 for item in downloads]
98 destinations = set(record.destination for record in records)
99 # Check for duplicate destinations, specifically for a local file path
100 # coming from two different sources. It's okay if there are duplicate
101 # destination records in the download request, but each of those must
102 # be specified by the same source url and have the same checksum.
103 #
104 # An easy quick check just asks if the set of destinations is smaller
105 # than the total number of requested downloads. It can't be larger.
106 # If it *is* smaller, then there are some duplicates, however the
107 # duplicates may be legitimate, so look at the details.
108 #
109 # Note though that we cannot pass duplicates destinations to udm, so we
110 # have to filter out legitimate duplicates. That's fine since they
111 # really are pointing to the same file, and will end up in the
112 # destination location.
113 if len(destinations) < len(downloads):
114 by_destination = dict()
115 unique_downloads = set()
116 for record in records:
117 by_destination.setdefault(record.destination, set()).add(
118 record)
119 unique_downloads.add(record)
120 duplicates = []
121 for dst, seen in by_destination.items():
122 if len(seen) > 1:
123 # Tuples will look better in the pretty-printed output.
124 duplicates.append(
125 (dst, sorted(tuple(dup) for dup in seen)))
126 if len(duplicates) > 0:
127 raise DuplicateDestinationError(sorted(duplicates))
128 # Uniquify the downloads.
129 records = list(unique_downloads)
130 return records
131
132 def _do_callback(self):
133 # Be defensive, so yes, use a bare except. If an exception occurs in
134 # the callback, log it, but continue onward.
135 for callback in self.callbacks:
136 try:
137 callback(self.received, self.total)
138 except:
139 log.exception('Exception in progress callback')
140
141 def cancel(self):
142 """Cancel any current downloads."""
143 self._queued_cancel = True
144
145 def pause(self):
146 """Pause the download, but only if one is in progress."""
147 pass # pragma: no cover
148
149 def resume(self):
150 """Resume the download, but only if one is in progress."""
151 pass # pragma: no cover
152
153 def _get_files(self, records, pausable):
154 raise NotImplementedError # pragma: no cover
177155
178 def get_files(self, downloads, *, pausable=False):156 def get_files(self, downloads, *, pausable=False):
179 """Download a bunch of files concurrently.157 """Download a bunch of files concurrently.
@@ -204,52 +182,16 @@
204 :raises: DuplicateDestinationError if more than one source url is182 :raises: DuplicateDestinationError if more than one source url is
205 downloaded to the same destination file.183 downloaded to the same destination file.
206 """184 """
207 assert self._iface is None
208 if self._queued_cancel:185 if self._queued_cancel:
209 # A cancel is queued, so don't actually download anything.186 # A cancel is queued, so don't actually download anything.
210 raise Canceled187 raise Canceled
211 if len(downloads) == 0:188 if len(downloads) == 0:
212 # Nothing to download. See LP: #1245597.189 # Nothing to download. See LP: #1245597.
213 return190 return
214 # Convert the downloads items to download records.191 records = self._get_download_records(downloads)
215 records = [item if isinstance(item, _RecordType) else Record(*item)192 # Better logging of the requested downloads. However, we want the
216 for item in downloads]193 # entire block of multiline log output to appear under a single
217 destinations = set(record.destination for record in records)194 # timestamp.
218 # Check for duplicate destinations, specifically for a local file path
219 # coming from two different sources. It's okay if there are duplicate
220 # destination records in the download request, but each of those must
221 # be specified by the same source url and have the same checksum.
222 #
223 # An easy quick check just asks if the set of destinations is smaller
224 # than the total number of requested downloads. It can't be larger.
225 # If it *is* smaller, then there are some duplicates, however the
226 # duplicates may be legitimate, so look at the details.
227 #
228 # Note though that we cannot pass duplicates destinations to udm,
229 # so we have to filter out legitimate duplicates. That's fine since
230 # they really are pointing to the same file, and will end up in the
231 # destination location.
232 if len(destinations) < len(downloads):
233 by_destination = dict()
234 unique_downloads = set()
235 for record in records:
236 by_destination.setdefault(record.destination, set()).add(
237 record)
238 unique_downloads.add(record)
239 duplicates = []
240 for dst, seen in by_destination.items():
241 if len(seen) > 1:
242 # Tuples will look better in the pretty-printed output.
243 duplicates.append(
244 (dst, sorted(tuple(dup) for dup in seen)))
245 if len(duplicates) > 0:
246 raise DuplicateDestinationError(sorted(duplicates))
247 # Uniquify the downloads.
248 records = list(unique_downloads)
249 bus = dbus.SystemBus()
250 service = bus.get_object(DOWNLOADER_INTERFACE, '/')
251 iface = dbus.Interface(service, MANAGER_INTERFACE)
252 # Better logging of the requested downloads.
253 fp = StringIO()195 fp = StringIO()
254 print('[0x{:x}] Requesting group download:'.format(id(self)), file=fp)196 print('[0x{:x}] Requesting group download:'.format(id(self)), file=fp)
255 for record in records:197 for record in records:
@@ -258,69 +200,38 @@
258 else:200 else:
259 print('\t{} [{}] -> {}'.format(*record), file=fp)201 print('\t{} [{}] -> {}'.format(*record), file=fp)
260 log.info('{}'.format(fp.getvalue()))202 log.info('{}'.format(fp.getvalue()))
261 object_path = iface.createDownloadGroup(203 self._get_files(records, pausable)
262 records,204
263 'sha256',205
264 False, # Don't allow GSM yet.206def get_download_manager(*args):
265 # https://bugs.freedesktop.org/show_bug.cgi?id=55594207 # We have to avoid circular imports since both download managers import
266 dbus.Dictionary(signature='sv'),208 # various things from this module.
267 _headers())209 from systemimage.curl import CurlDownloadManager
268 download = bus.get_object(OBJECT_NAME, object_path)210 from systemimage.udm import DOWNLOADER_INTERFACE, UDMDownloadManager
269 self._iface = dbus.Interface(download, OBJECT_INTERFACE)211 # Detect if we have ubuntu-download-manager.
270 # Are GSM downloads allowed? Yes, except if auto_download is set to 1212 #
271 # (i.e. wifi-only).213 # Use PyCURL based downloader if no udm is found, or if the environment
272 allow_gsm = Settings().get('auto_download') != '1'214 # variable is set. However, if we're told to use PyCURL and it's
273 DBusDownloadManager._set_gsm(self._iface, allow_gsm=allow_gsm)215 # unavailable, throw an exception.
274 # Start the download.216 cls = None
275 reactor = DownloadReactor(bus, self.callback, pausable)217 use_pycurl = os.environ.get('SYSTEMIMAGE_PYCURL')
276 reactor.schedule(self._iface.start)218 if use_pycurl is None:
277 log.info('[0x{:x}] Running group download reactor', id(self))219 # Auto-detect. For backward compatibility, use udm if it's available,
278 reactor.run()220 # otherwise use PyCURL.
279 # This download is complete so the object path is no longer221 try:
280 # applicable. Setting this to None will cause subsequent cancels to222 bus = dbus.SystemBus()
281 # be queued.223 bus.get_object(DOWNLOADER_INTERFACE, '/')
282 self._iface = None224 udm_available = True
283 log.info('[0x{:x}] Group download reactor done', id(self))225 except dbus.exceptions.DBusException:
284 if reactor.error is not None:226 udm_available = False
285 log.error('Reactor error: {}'.format(reactor.error))227 if udm_available:
286 if reactor.canceled:228 cls = UDMDownloadManager
287 log.info('Reactor canceled')229 elif pycurl is None:
288 # Report any other problems.230 raise ImportError('No module named {}'.format('pycurl'))
289 if reactor.error is not None:
290 raise FileNotFoundError(reactor.error)
291 if reactor.canceled:
292 raise Canceled
293 if reactor.timed_out:
294 raise TimeoutError
295 # For sanity.
296 for record in records:
297 assert os.path.exists(record.destination), (
298 'Missing destination: {}'.format(record))
299
300 @staticmethod
301 def _set_gsm(iface, *, allow_gsm):
302 # This is a separate method for easier testing via mocks.
303 iface.allowGSMDownload(allow_gsm)
304
305 def cancel(self):
306 """Cancel any current downloads."""
307 if self._iface is None:
308 # Since there's no download in progress right now, there's nothing
309 # to cancel. Setting this flag queues the cancel signal once the
310 # reactor starts running again. Yes, this is a bit weird, but if
311 # we don't do it this way, the caller will immediately get a
312 # Canceled exception, which isn't helpful because it's expecting
313 # one when the next download begins.
314 self._queued_cancel = True
315 else:231 else:
316 self._iface.cancel()232 cls = CurlDownloadManager
317233 else:
318 def pause(self):234 cls = (CurlDownloadManager
319 """Pause the download, but only if one is in progress."""235 if use_pycurl.lower() in ('1', 'yes', 'true')
320 if self._iface is not None: # pragma: no branch236 else UDMDownloadManager)
321 self._iface.pause()237 return cls(*args)
322
323 def resume(self):
324 """Resume the download, but only if one is in progress."""
325 if self._iface is not None: # pragma: no branch
326 self._iface.resume()
327238
=== modified file 'systemimage/gpg.py'
--- systemimage/gpg.py 2014-09-17 13:41:31 +0000
+++ systemimage/gpg.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -188,6 +188,11 @@
188 :type data_path: str188 :type data_path: str
189 :return: bool189 :return: bool
190 """190 """
191 # For testing on some systems that are connecting to test servers, GPG
192 # verification isn't possible. The s-i-cli supports a switch to
193 # disable all GPG checks.
194 if config.skip_gpg_verification:
195 return True
191 with open(signature_path, 'rb') as sig_fp:196 with open(signature_path, 'rb') as sig_fp:
192 verified = self._ctx.verify_file(sig_fp, data_path)197 verified = self._ctx.verify_file(sig_fp, data_path)
193 # If the file is properly signed, we'll be able to get back a set of198 # If the file is properly signed, we'll be able to get back a set of
194199
=== modified file 'systemimage/helpers.py'
--- systemimage/helpers.py 2014-09-17 13:41:31 +0000
+++ systemimage/helpers.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -20,6 +20,8 @@
20 'MiB',20 'MiB',
21 'as_loglevel',21 'as_loglevel',
22 'as_object',22 'as_object',
23 'as_port',
24 'as_stripped',
23 'as_timedelta',25 'as_timedelta',
24 'atomic',26 'atomic',
25 'calculate_signature',27 'calculate_signature',
@@ -34,7 +36,6 @@
3436
35import os37import os
36import re38import re
37import time
38import random39import random
39import shutil40import shutil
40import logging41import logging
@@ -46,11 +47,12 @@
46from importlib import import_module47from importlib import import_module
4748
4849
50UNIQUE_MACHINE_ID_FILES = ['/var/lib/dbus/machine-id', '/etc/machine-id']
49LAST_UPDATE_FILE = '/userdata/.last_update'51LAST_UPDATE_FILE = '/userdata/.last_update'
50UNIQUE_MACHINE_ID_FILE = '/var/lib/dbus/machine-id'
51DEFAULT_DIRMODE = 0o0270052DEFAULT_DIRMODE = 0o02700
52MiB = 1 << 2053MiB = 1 << 20
53EMPTYSTRING = ''54EMPTYSTRING = ''
55NO_PORT = object()
5456
5557
56def calculate_signature(fp, hash_class=None):58def calculate_signature(fp, hash_class=None):
@@ -79,7 +81,7 @@
79 """Like os.remove() but don't complain if the file doesn't exist."""81 """Like os.remove() but don't complain if the file doesn't exist."""
80 try:82 try:
81 os.remove(path)83 os.remove(path)
82 except (FileNotFoundError, IsADirectoryError):84 except (FileNotFoundError, IsADirectoryError, PermissionError):
83 pass85 pass
8486
8587
@@ -195,13 +197,26 @@
195 dbus = 'ERROR'197 dbus = 'ERROR'
196 main_level = getattr(logging, main, None)198 main_level = getattr(logging, main, None)
197 if main_level is None or not isinstance(main_level, int):199 if main_level is None or not isinstance(main_level, int):
198 raise ValueError200 raise ValueError(value)
199 dbus_level = getattr(logging, dbus, None)201 dbus_level = getattr(logging, dbus, None)
200 if dbus_level is None or not isinstance(dbus_level, int):202 if dbus_level is None or not isinstance(dbus_level, int):
201 raise ValueError203 raise ValueError(value)
202 return main_level, dbus_level204 return main_level, dbus_level
203205
204206
207def as_port(value):
208 if value.lower() in ('disabled', 'disable'):
209 return NO_PORT
210 result = int(value)
211 if result < 0:
212 raise ValueError(value)
213 return result
214
215
216def as_stripped(value):
217 return value.strip()
218
219
205@contextmanager220@contextmanager
206def temporary_directory(*args, **kws):221def temporary_directory(*args, **kws):
207 """A context manager that creates a temporary directory.222 """A context manager that creates a temporary directory.
@@ -227,30 +242,24 @@
227def last_update_date():242def last_update_date():
228 """Return the last update date.243 """Return the last update date.
229244
230 Taken from the mtime of the following files, in order:245 If /userdata/.last_update exists, we use this file's mtime. If it doesn't
231246 exist, then we use the latest mtime of any of the files in
232 - /userdata/.last_update247 /etc/system-image/config.d/*.ini (or whatever directory was given with the
233 - /etc/system-image/channel.ini248 -C/--config option).
234 - /etc/ubuntu-build
235
236 First existing path wins.
237 """249 """
238 # Avoid circular imports.250 # Avoid circular imports.
239 from systemimage.config import config251 from systemimage.config import config
240 channel_ini = os.path.join(252 try:
241 os.path.dirname(config.config_file), 'channel.ini')253 timestamp = datetime.fromtimestamp(os.stat(LAST_UPDATE_FILE).st_mtime)
242 ubuntu_build = config.system.build_file254 except (FileNotFoundError, PermissionError):
243 for path in (LAST_UPDATE_FILE, channel_ini, ubuntu_build):255 # We fall back to the latest mtime of the config.d/*.ini files.
244 try:256 timestamps = sorted(
245 # Local time, since we can't know the timezone.257 datetime.fromtimestamp(path.stat().st_mtime)
246 timestamp = datetime.fromtimestamp(os.stat(path).st_mtime)258 for path in config.ini_files)
247 # Seconds resolution.259 if len(timestamps) == 0:
248 timestamp = timestamp.replace(microsecond=0)260 return 'Unknown'
249 return str(timestamp)261 timestamp = timestamps[-1]
250 except (FileNotFoundError, PermissionError):262 return str(timestamp.replace(microsecond=0))
251 pass
252 else:
253 return 'Unknown'
254263
255264
256def version_detail(details_string=None):265def version_detail(details_string=None):
@@ -270,19 +279,20 @@
270 return details279 return details
271280
272281
273_pp_cache = None282def phased_percentage(channel, target):
274283 # Avoid circular imports.
275def phased_percentage(*, reset=False):284 from systemimage.config import config
276 global _pp_cache285 if config.phase_override is not None:
277 if _pp_cache is None:286 return config.phase_override
278 with open(UNIQUE_MACHINE_ID_FILE, 'rb') as fp:287 for path in UNIQUE_MACHINE_ID_FILES:
279 data = fp.read()288 try:
280 now = str(time.time()).encode('us-ascii')289 with open(path, 'r', encoding='utf-8') as fp:
281 r = random.Random()290 machine_id = fp.read().strip()
282 r.seed(data + now)291 break # pragma: no branch
283 _pp_cache = r.randint(0, 100)292 except FileNotFoundError:
284 try:293 pass
285 return _pp_cache294 else:
286 finally:295 raise RuntimeError('No machine-id file found')
287 if reset:296 r = random.Random()
288 _pp_cache = None297 r.seed('{}.{}.{}'.format(channel, target, machine_id))
298 return r.randint(0, 100)
289299
=== modified file 'systemimage/image.py'
--- systemimage/image.py 2014-09-17 13:41:31 +0000
+++ systemimage/image.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -74,3 +74,7 @@
74 @property74 @property
75 def phased_percentage(self):75 def phased_percentage(self):
76 return self.__untranslated__.get('phased-percentage', 100)76 return self.__untranslated__.get('phased-percentage', 100)
77
78 @property
79 def version_detail(self):
80 return self.__untranslated__.get('version_detail', '')
7781
=== modified file 'systemimage/index.py'
--- systemimage/index.py 2014-02-20 23:03:24 +0000
+++ systemimage/index.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -24,7 +24,6 @@
2424
25from datetime import datetime, timezone25from datetime import datetime, timezone
26from systemimage.bag import Bag26from systemimage.bag import Bag
27from systemimage.helpers import phased_percentage
28from systemimage.image import Image27from systemimage.image import Image
2928
3029
@@ -49,7 +48,6 @@
49 global_ = Bag(generated_at=generated_at)48 global_ = Bag(generated_at=generated_at)
50 # Parse the images.49 # Parse the images.
51 images = []50 images = []
52 percentage = phased_percentage()
53 for image_data in mapping['images']:51 for image_data in mapping['images']:
54 # Descriptions can be any of:52 # Descriptions can be any of:
55 #53 #
@@ -70,6 +68,5 @@
70 image = Image(files=bundles,68 image = Image(files=bundles,
71 descriptions=descriptions,69 descriptions=descriptions,
72 **image_data)70 **image_data)
73 if percentage <= image.phased_percentage:71 images.append(image)
74 images.append(image)
75 return cls(global_=global_, images=images)72 return cls(global_=global_, images=images)
7673
=== modified file 'systemimage/keyring.py'
--- systemimage/keyring.py 2014-02-20 23:03:24 +0000
+++ systemimage/keyring.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -30,7 +30,7 @@
30from contextlib import ExitStack30from contextlib import ExitStack
31from datetime import datetime, timezone31from datetime import datetime, timezone
32from systemimage.config import config32from systemimage.config import config
33from systemimage.download import DBusDownloadManager33from systemimage.download import get_download_manager
34from systemimage.gpg import Context34from systemimage.gpg import Context
35from systemimage.helpers import makedirs, safe_remove35from systemimage.helpers import makedirs, safe_remove
36from urllib.parse import urljoin36from urllib.parse import urljoin
@@ -86,8 +86,8 @@
86 else:86 else:
87 srcurl = urls87 srcurl = urls
88 ascurl = urls + '.asc'88 ascurl = urls + '.asc'
89 tarxz_src = urljoin(config.service.https_base, srcurl)89 tarxz_src = urljoin(config.https_base, srcurl)
90 ascxz_src = urljoin(config.service.https_base, ascurl)90 ascxz_src = urljoin(config.https_base, ascurl)
91 # Calculate the local paths to the temporary download files. The91 # Calculate the local paths to the temporary download files. The
92 # blacklist goes to the data partition and all the other files go to the92 # blacklist goes to the data partition and all the other files go to the
93 # cache partition.93 # cache partition.
@@ -102,7 +102,7 @@
102 safe_remove(ascxz_dst)102 safe_remove(ascxz_dst)
103 with ExitStack() as stack:103 with ExitStack() as stack:
104 # Let FileNotFoundError percolate up.104 # Let FileNotFoundError percolate up.
105 DBusDownloadManager().get_files([105 get_download_manager().get_files([
106 (tarxz_src, tarxz_dst),106 (tarxz_src, tarxz_dst),
107 (ascxz_src, ascxz_dst),107 (ascxz_src, ascxz_dst),
108 ])108 ])
109109
=== modified file 'systemimage/logging.py'
--- systemimage/logging.py 2014-09-17 13:41:31 +0000
+++ systemimage/logging.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -80,7 +80,9 @@
80def initialize(*, verbosity=0):80def initialize(*, verbosity=0):
81 """Initialize the loggers."""81 """Initialize the loggers."""
82 main, dbus = config.system.loglevel82 main, dbus = config.system.loglevel
83 for name, loglevel in (('systemimage', main), ('systemimage.dbus', dbus)):83 for name, loglevel in (('systemimage', main),
84 ('systemimage.dbus', dbus),
85 ('dbus.proxies', dbus)):
84 level = {86 level = {
85 0: logging.ERROR,87 0: logging.ERROR,
86 1: logging.INFO,88 1: logging.INFO,
8789
=== modified file 'systemimage/main.py'
--- systemimage/main.py 2014-09-26 14:36:34 +0000
+++ systemimage/main.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -21,18 +21,19 @@
21 ]21 ]
2222
2323
24import os
25import sys24import sys
25import json
26import logging26import logging
27import argparse27import argparse
2828
29from dbus.mainloop.glib import DBusGMainLoop29from dbus.mainloop.glib import DBusGMainLoop
30from pkg_resources import resource_string as resource_bytes30from pkg_resources import resource_string as resource_bytes
31from systemimage.apply import factory_reset, production_reset
31from systemimage.candidates import delta_filter, full_filter32from systemimage.candidates import delta_filter, full_filter
32from systemimage.config import config33from systemimage.config import config
33from systemimage.helpers import last_update_date, makedirs, version_detail34from systemimage.helpers import (
35 last_update_date, makedirs, phased_percentage, version_detail)
34from systemimage.logging import initialize36from systemimage.logging import initialize
35from systemimage.reboot import factory_reset
36from systemimage.settings import Settings37from systemimage.settings import Settings
37from systemimage.state import State38from systemimage.state import State
38from textwrap import dedent39from textwrap import dedent
@@ -41,12 +42,47 @@
41__version__ = resource_bytes(42__version__ = resource_bytes(
42 'systemimage', 'version.txt').decode('utf-8').strip()43 'systemimage', 'version.txt').decode('utf-8').strip()
4344
44DEFAULT_CONFIG_FILE = '/etc/system-image/client.ini'45DEFAULT_CONFIG_D = '/etc/system-image/config.d'
45COLON = ':'46COLON = ':'
47LINE_LENGTH = 78
48
49
50class _DotsProgress:
51 def __init__(self):
52 self._dot_count = 0
53
54 def callback(self, received, total):
55 received = int(received)
56 total = int(total)
57 sys.stderr.write('.')
58 sys.stderr.flush()
59 self._dot_count += 1
60 show_dots = self._dot_count % LINE_LENGTH == 0
61 if show_dots or received >= total: # pragma: no cover
62 sys.stderr.write('\n')
63 sys.stderr.flush()
64
65
66class _LogfileProgress:
67 def __init__(self, log):
68 self._log = log
69
70 def callback(self, received, total):
71 self._log.debug('received: {} of {} bytes', received, total)
72
73
74def _json_progress(received, total):
75 # For use with --progress=json output. LP: #1423622
76 message = json.dumps(dict(
77 type='progress',
78 now=received,
79 total=total))
80 sys.stdout.write(message)
81 sys.stdout.write('\n')
82 sys.stdout.flush()
4683
4784
48def main():85def main():
49 global config
50 parser = argparse.ArgumentParser(86 parser = argparse.ArgumentParser(
51 prog='system-image-cli',87 prog='system-image-cli',
52 description='Ubuntu System Image Upgrader')88 description='Ubuntu System Image Upgrader')
@@ -54,10 +90,10 @@
54 action='version',90 action='version',
55 version='system-image-cli {}'.format(__version__))91 version='system-image-cli {}'.format(__version__))
56 parser.add_argument('-C', '--config',92 parser.add_argument('-C', '--config',
57 default=DEFAULT_CONFIG_FILE, action='store',93 default=DEFAULT_CONFIG_D, action='store',
58 metavar='FILE',94 metavar='DIRECTORY',
59 help="""Use the given configuration file instead of95 help="""Use the given configuration directory instead
60 the default""")96 of the default""")
61 parser.add_argument('-b', '--build',97 parser.add_argument('-b', '--build',
62 default=None, action='store',98 default=None, action='store',
63 help="""Override the current build number just99 help="""Override the current build number just
@@ -76,12 +112,16 @@
76 full updates or only delta updates. The112 full updates or only delta updates. The
77 argument to this option must be either `full`113 argument to this option must be either `full`
78 or `delta`""")114 or `delta`""")
79 parser.add_argument('-g', '--no-reboot',115 parser.add_argument('-g', '--no-apply',
80 default=False, action='store_true',116 default=False, action='store_true',
81 help="""Download (i.e. "get") all the data files and117 help="""Download (i.e. "get") all the data files and
82 prepare for updating, but don't actually118 prepare for updating, but don't actually
83 reboot the device into recovery to apply the119 reboot the device into recovery to apply the
84 update""")120 update""")
121 # Deprecated since si 3.0.
122 parser.add_argument('--no-reboot',
123 default=False, action='store_true',
124 help="""Deprecated; use -g/--no-apply""")
85 parser.add_argument('-i', '--info',125 parser.add_argument('-i', '--info',
86 default=False, action='store_true',126 default=False, action='store_true',
87 help="""Show some information about the current127 help="""Show some information about the current
@@ -94,6 +134,15 @@
94 parser.add_argument('-v', '--verbose',134 parser.add_argument('-v', '--verbose',
95 default=0, action='count',135 default=0, action='count',
96 help='Increase verbosity')136 help='Increase verbosity')
137 parser.add_argument('--progress',
138 default=[], action='append',
139 help="""Add a progress meter. Available meters are:
140 dots, logfile, and json. Multiple --progress
141 options are allowed.""")
142 parser.add_argument('-p', '--percentage',
143 default=None, action='store',
144 help="""Override the device's phased percentage value
145 during upgrade candidate calculation.""")
97 parser.add_argument('--list-channels',146 parser.add_argument('--list-channels',
98 default=False, action='store_true',147 default=False, action='store_true',
99 help="""List all available channels, then exit""")148 help="""List all available channels, then exit""")
@@ -102,6 +151,12 @@
102 help="""Perform a destructive factory reset and151 help="""Perform a destructive factory reset and
103 reboot. WARNING: this will wipe all user data152 reboot. WARNING: this will wipe all user data
104 on the device!""")153 on the device!""")
154 parser.add_argument('--production-reset',
155 default=False, action='store_true',
156 help="""Perform a destructive production reset
157 (similar to factory reset) and reboot.
158 WARNING: this will wipe all user data
159 on the device!""")
105 parser.add_argument('--switch',160 parser.add_argument('--switch',
106 default=None, action='store', metavar='CHANNEL',161 default=None, action='store', metavar='CHANNEL',
107 help="""Switch to the given channel. This is162 help="""Switch to the given channel. This is
@@ -128,27 +183,36 @@
128 help="""Delete the key and its value. It is a no-op183 help="""Delete the key and its value. It is a no-op
129 if the key does not exist. Multiple184 if the key does not exist. Multiple
130 --del arguments can be given.""")185 --del arguments can be given.""")
186 # Hidden system-image-cli only feature for testing purposes. LP: #1333414
187 parser.add_argument('--skip-gpg-verification',
188 default=False, action='store_true',
189 help=argparse.SUPPRESS)
131190
132 args = parser.parse_args(sys.argv[1:])191 args = parser.parse_args(sys.argv[1:])
133 try:192 try:
134 config.load(args.config)193 config.load(args.config)
135 except FileNotFoundError as error:194 except (TypeError, FileNotFoundError):
136 parser.error('\nConfiguration file not found: {}'.format(error))195 parser.error('\nConfiguration directory not found: {}'.format(
196 args.config))
137 assert 'parser.error() does not return' # pragma: no cover197 assert 'parser.error() does not return' # pragma: no cover
138 # Load the optional channel.ini file, which must live next to the198
139 # configuration file. It's okay if this file does not exist.199 if args.skip_gpg_verification:
140 channel_ini = os.path.join(os.path.dirname(args.config), 'channel.ini')200 print("""\
141 try:201WARNING: All GPG signature verifications have been disabled.
142 config.load(channel_ini, override=True)202Your upgrades are INSECURE.""", file=sys.stderr)
143 except FileNotFoundError:203 config.skip_gpg_verification = True
144 pass204
145205 # Perform factory and production resets.
146 # Perform a factory reset.
147 if args.factory_reset:206 if args.factory_reset:
148 factory_reset()207 factory_reset()
149 # We should never get here, except possibly during the testing208 # We should never get here, except possibly during the testing
150 # process, so just return as normal.209 # process, so just return as normal.
151 return 0210 return 0
211 if args.production_reset:
212 production_reset()
213 # We should never get here, except possibly during the testing
214 # process, so just return as normal.
215 return 0
152216
153 # Handle all settings arguments. They are mutually exclusive.217 # Handle all settings arguments. They are mutually exclusive.
154 if sum(bool(arg) for arg in218 if sum(bool(arg) for arg in
@@ -213,6 +277,8 @@
213 config.channel = args.channel277 config.channel = args.channel
214 if args.device is not None:278 if args.device is not None:
215 config.device = args.device279 config.device = args.device
280 if args.percentage is not None:
281 config.phase_override = args.percentage
216282
217 if args.info:283 if args.info:
218 alias = getattr(config.service, 'channel_target', None)284 alias = getattr(config.service, 'channel_target', None)
@@ -245,11 +311,16 @@
245 print('version {}: {}'.format(key, details[key]))311 print('version {}: {}'.format(key, details[key]))
246 return 0312 return 0
247313
314 DBusGMainLoop(set_as_default=True)
315
248 if args.list_channels:316 if args.list_channels:
249 state = State()317 state = State()
250 try:318 try:
251 state.run_thru('get_channel')319 state.run_thru('get_channel')
252 except Exception:320 except Exception:
321 print('Exception occurred during channel search; '
322 'see log file for details',
323 file=sys.stderr)
253 log.exception('system-image-cli exception')324 log.exception('system-image-cli exception')
254 return 1325 return 1
255 print('Available channels:')326 print('Available channels:')
@@ -261,33 +332,26 @@
261 print(' {} (alias for: {})'.format(key, alias))332 print(' {} (alias for: {})'.format(key, alias))
262 return 0333 return 0
263334
264 # When verbosity is at 1, logging every progress signal from
265 # ubuntu-download-manager would be way too noisy. OTOH, not printing
266 # anything leads some folks to think the process is just hung, since it
267 # can take a long time to download all the data files. As a compromise,
268 # we'll output some dots to stderr at verbosity 1, but we won't log these
269 # dots since they would just be noise. This doesn't have to be perfect.
270 if args.verbose == 1: # pragma: no cover
271 dot_count = 0
272 def callback(received, total):
273 nonlocal dot_count
274 sys.stderr.write('.')
275 sys.stderr.flush()
276 dot_count += 1
277 if dot_count % 78 == 0 or received >= total:
278 sys.stderr.write('\n')
279 sys.stderr.flush()
280 else:
281 def callback(received, total):
282 log.debug('received: {} of {} bytes', received, total)
283
284 DBusGMainLoop(set_as_default=True)
285 state = State(candidate_filter=candidate_filter)335 state = State(candidate_filter=candidate_filter)
286 state.downloader.callback = callback336
337 for meter in args.progress:
338 if meter == 'dots':
339 state.downloader.callbacks.append(_DotsProgress().callback)
340 elif meter == 'json':
341 state.downloader.callbacks.append(_json_progress)
342 elif meter == 'logfile':
343 state.downloader.callbacks.append(_LogfileProgress(log).callback)
344 else:
345 parser.error('Unknown progress meter: {}'.format(meter))
346 assert 'parser.error() does not return' # pragma: no cover
347
287 if args.dry_run:348 if args.dry_run:
288 try:349 try:
289 state.run_until('download_files')350 state.run_until('download_files')
290 except Exception:351 except Exception:
352 print('Exception occurred during dry-run; '
353 'see log file for details',
354 file=sys.stderr)
291 log.exception('system-image-cli exception')355 log.exception('system-image-cli exception')
292 return 1356 return 1
293 # Say -c <no-such-channel> was given. This will fail.357 # Say -c <no-such-channel> was given. This will fail.
@@ -296,16 +360,20 @@
296 else:360 else:
297 winning_path = [str(image.version) for image in state.winner]361 winning_path = [str(image.version) for image in state.winner]
298 kws = dict(path=COLON.join(winning_path))362 kws = dict(path=COLON.join(winning_path))
363 target_build = state.winner[-1].version
299 if state.channel_switch is None:364 if state.channel_switch is None:
300 # We're not switching channels due to an alias change.365 # We're not switching channels due to an alias change.
301 template = 'Upgrade path is {path}'366 template = 'Upgrade path is {path}'
367 percentage = phased_percentage(config.channel, target_build)
302 else:368 else:
303 # This upgrade changes the channel that our alias is mapped369 # This upgrade changes the channel that our alias is mapped
304 # to, so include that information in the output.370 # to, so include that information in the output.
305 template = 'Upgrade path is {path} ({from} -> {to})'371 template = 'Upgrade path is {path} ({from} -> {to})'
306 kws['from'], kws['to'] = state.channel_switch372 kws['from'], kws['to'] = state.channel_switch
373 percentage = phased_percentage(kws['to'], target_build)
307 print(template.format(**kws))374 print(template.format(**kws))
308 return375 print('Target phase: {}%'.format(percentage))
376 return 0
309 else:377 else:
310 # Run the state machine to conclusion. Suppress all exceptions, but378 # Run the state machine to conclusion. Suppress all exceptions, but
311 # note that the state machine will log them. If an exception occurs,379 # note that the state machine will log them. If an exception occurs,
@@ -313,13 +381,15 @@
313 log.info('running state machine [{}/{}]',381 log.info('running state machine [{}/{}]',
314 config.channel, config.device)382 config.channel, config.device)
315 try:383 try:
316 if args.no_reboot:384 if args.no_apply or args.no_reboot:
317 state.run_until('reboot')385 state.run_until('apply')
318 else:386 else:
319 list(state)387 list(state)
320 except KeyboardInterrupt: # pragma: no cover388 except KeyboardInterrupt: # pragma: no cover
321 return 0389 return 0
322 except Exception:390 except Exception:
391 print('Exception occurred during update; see log file for details',
392 file=sys.stderr)
323 log.exception('system-image-cli exception')393 log.exception('system-image-cli exception')
324 return 1394 return 1
325 else:395 else:
326396
=== modified file 'systemimage/reactor.py'
--- systemimage/reactor.py 2014-09-17 13:41:31 +0000
+++ systemimage/reactor.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -76,11 +76,14 @@
76 self._quitter = GLib.timeout_add_seconds(76 self._quitter = GLib.timeout_add_seconds(
77 self._active_timeout, self._quit_with_error)77 self._active_timeout, self._quit_with_error)
7878
79 def react_to(self, signal):79 def react_to(self, signal, object_path=None):
80 signal_match = self._bus.add_signal_receiver(80 signal_match = self._bus.add_signal_receiver(
81 self._handle_signal, signal_name=signal,81 self._handle_signal,
82 signal_name=signal,
83 path=object_path,
82 member_keyword='member',84 member_keyword='member',
83 path_keyword='path')85 path_keyword='path',
86 )
84 self._signal_matches.append(signal_match)87 self._signal_matches.append(signal_match)
8588
86 def schedule(self, method, milliseconds=50):89 def schedule(self, method, milliseconds=50):
8790
=== modified file 'systemimage/scores.py'
--- systemimage/scores.py 2014-09-17 13:41:31 +0000
+++ systemimage/scores.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -26,19 +26,18 @@
2626
27import logging27import logging
2828
29from io import StringIO
30from itertools import count29from itertools import count
31from systemimage.helpers import MiB30from systemimage.helpers import MiB, phased_percentage
31
3232
33log = logging.getLogger('systemimage')33log = logging.getLogger('systemimage')
34
35COLON = ':'34COLON = ':'
3635
3736
38class Scorer:37class Scorer:
39 """Abstract base class providing an API for candidate selection."""38 """Abstract base class providing an API for candidate selection."""
4039
41 def choose(self, candidates):40 def choose(self, candidates, channel):
42 """Choose the candidate upgrade paths.41 """Choose the candidate upgrade paths.
4342
44 Lowest score wins.43 Lowest score wins.
@@ -47,10 +46,14 @@
47 the device from the current version to the latest version, sorted46 the device from the current version to the latest version, sorted
48 in order from oldest verson to newest.47 in order from oldest verson to newest.
49 :type candidates: list of lists48 :type candidates: list of lists
49 :param channel: The channel being upgraded to. This is used in the
50 phased update calculate.
51 :type channel: str
50 :return: The chosen path.52 :return: The chosen path.
51 :rtype: list53 :rtype: list
52 """54 """
53 if len(candidates) == 0:55 if len(candidates) == 0:
56 log.debug('No candidates, so no winner')
54 return []57 return []
55 # We want to zip together the score for each candidate path, plus the58 # We want to zip together the score for each candidate path, plus the
56 # candidate path, so that when we sort the sequence, we'll always get59 # candidate path, so that when we sort the sequence, we'll always get
@@ -68,17 +71,31 @@
68 # Be sure that after all is said and done we return the list of Images71 # Be sure that after all is said and done we return the list of Images
69 # though!72 # though!
70 scores = sorted(zip(self.score(candidates), count(), candidates))73 scores = sorted(zip(self.score(candidates), count(), candidates))
71 fp = StringIO()74 # Calculate the phase percentage for the device. Use the highest
72 print('{} path scores (last one wins):'.format(75 # available build number as input into the random seed.
73 self.__class__.__name__),76 max_target_number = -1
74 file=fp)77 for score, i, path in scores:
75 for score, i, candidate in reversed(scores):78 # The last image will be the target image.
76 print('\t[{:4d}] -> {}'.format(79 assert len(path) > 0, 'Empty upgrade candidate path?'
80 max_target_number = max(max_target_number, path[-1].version)
81 assert max_target_number != -1, 'No max target version?'
82 device_percentage = phased_percentage(channel, max_target_number)
83 log.debug('Device phased percentage: {}%'.format(device_percentage))
84 log.debug('{} path scores:'.format(self.__class__.__name__))
85 # Log the candidate paths, their scores, and their phases.
86 for score, i, path in reversed(scores):
87 log.debug('\t[{:4d}] -> {} ({}%)'.format(
77 score,88 score,
78 COLON.join(str(image.version) for image in candidate)),89 COLON.join(str(image.version) for image in path),
79 file=fp)90 (path[-1].phased_percentage if len(path) > 0 else '--')
80 log.debug('{}'.format(fp.getvalue()))91 ))
81 return scores[0][2]92 for score, i, path in scores:
93 image_percentage = path[-1].phased_percentage
94 # An image percentage of 0 means that it's been pulled.
95 if image_percentage > 0 and device_percentage <= image_percentage:
96 return path
97 # No upgrade path.
98 return []
8299
83 def score(self, candidates): # pragma: no cover100 def score(self, candidates): # pragma: no cover
84 """Like `choose()` except returns the candidate path scores.101 """Like `choose()` except returns the candidate path scores.
85102
=== modified file 'systemimage/service.py'
--- systemimage/service.py 2014-09-17 13:41:31 +0000
+++ systemimage/service.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -20,7 +20,6 @@
20 ]20 ]
2121
2222
23import os
24import sys23import sys
25import dbus24import dbus
26import logging25import logging
@@ -33,7 +32,8 @@
33from systemimage.dbus import Loop32from systemimage.dbus import Loop
34from systemimage.helpers import makedirs33from systemimage.helpers import makedirs
35from systemimage.logging import initialize34from systemimage.logging import initialize
36from systemimage.main import DEFAULT_CONFIG_FILE35from systemimage.main import DEFAULT_CONFIG_D
36
3737
38# --testing is only enabled when the systemimage.testing package is38# --testing is only enabled when the systemimage.testing package is
39# available. This will be the case for the upstream source package, and when39# available. This will be the case for the upstream source package, and when
@@ -60,32 +60,28 @@
60 action='version',60 action='version',
61 version='system-image-dbus {}'.format(__version__))61 version='system-image-dbus {}'.format(__version__))
62 parser.add_argument('-C', '--config',62 parser.add_argument('-C', '--config',
63 default=DEFAULT_CONFIG_FILE, action='store',63 default=DEFAULT_CONFIG_D, action='store',
64 metavar='FILE',64 metavar='DIRECTORY',
65 help="""Use the given configuration file instead of65 help="""Use the given configuration directory instead
66 the default""")66 of the default""")
67 parser.add_argument('-v', '--verbose',67 parser.add_argument('-v', '--verbose',
68 default=0, action='count',68 default=0, action='count',
69 help='Increase verbosity')69 help='Increase verbosity')
70 # Hidden argument for special setup required by test environment.70 # Hidden argument for special setup required by test environment.
71 if instrument is not None: # pragma: no branch71 if instrument is not None: # pragma: no branch
72 parser.add_argument('--testing',72 parser.add_argument('--testing',
73 default=False, action='store',73 default=None, action='store',
74 help=argparse.SUPPRESS)
75 parser.add_argument('--self-signed-cert',
76 default=None, action='store',
74 help=argparse.SUPPRESS)77 help=argparse.SUPPRESS)
7578
76 args = parser.parse_args(sys.argv[1:])79 args = parser.parse_args(sys.argv[1:])
77 try:80 try:
78 config.load(args.config)81 config.load(args.config)
79 except FileNotFoundError as error:82 except TypeError as error:
80 parser.error('\nConfiguration file not found: {}'.format(error))83 parser.error('\nConfiguration directory not found: {}'.format(error))
81 assert 'parser.error() does not return' # pragma: no cover84 assert 'parser.error() does not return' # pragma: no cover
82 # Load the optional channel.ini file, which must live next to the
83 # configuration file. It's okay if this file does not exist.
84 channel_ini = os.path.join(os.path.dirname(args.config), 'channel.ini')
85 try:
86 config.load(channel_ini, override=True)
87 except FileNotFoundError:
88 pass
8985
90 # Create the temporary directory if it doesn't exist.86 # Create the temporary directory if it doesn't exist.
91 makedirs(config.system.tempdir)87 makedirs(config.system.tempdir)
@@ -112,12 +108,13 @@
112 loop = Loop()108 loop = Loop()
113 testing_mode = getattr(args, 'testing', None)109 testing_mode = getattr(args, 'testing', None)
114 if testing_mode:110 if testing_mode:
115 instrument(config, stack)111 instrument(config, stack, args.self_signed_cert)
116 config.dbus_service = get_service(112 config.dbus_service = get_service(
117 testing_mode, system_bus, '/Service', loop)113 testing_mode, system_bus, '/Service', loop)
118 else:114 else:
119 from systemimage.dbus import Service115 from systemimage.dbus import Service
120 config.dbus_service = Service(system_bus, '/Service', loop)116 config.dbus_service = Service(system_bus, '/Service', loop)
117
121 try:118 try:
122 loop.run()119 loop.run()
123 except KeyboardInterrupt: # pragma: no cover120 except KeyboardInterrupt: # pragma: no cover
124121
=== modified file 'systemimage/settings.py'
--- systemimage/settings.py 2014-09-17 13:41:31 +0000
+++ systemimage/settings.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
55
=== modified file 'systemimage/state.py'
--- systemimage/state.py 2014-09-17 13:41:31 +0000
+++ systemimage/state.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -35,7 +35,7 @@
35from systemimage.candidates import get_candidates, iter_path35from systemimage.candidates import get_candidates, iter_path
36from systemimage.channel import Channels36from systemimage.channel import Channels
37from systemimage.config import config37from systemimage.config import config
38from systemimage.download import DBusDownloadManager, Record38from systemimage.download import Record, get_download_manager
39from systemimage.gpg import Context, SignatureError39from systemimage.gpg import Context, SignatureError
40from systemimage.helpers import (40from systemimage.helpers import (
41 atomic, calculate_signature, makedirs, safe_remove, temporary_directory)41 atomic, calculate_signature, makedirs, safe_remove, temporary_directory)
@@ -58,6 +58,9 @@
58 self.got = got58 self.got = got
59 self.expected = checksum59 self.expected = checksum
6060
61 def __str__(self): # pragma: no cover
62 return 'got:{0.got} != exp:{0.expected}: {0.destination}'.format(self)
63
6164
62def _copy_if_missing(src, dstdir):65def _copy_if_missing(src, dstdir):
63 dst_path = os.path.join(dstdir, os.path.basename(src))66 dst_path = os.path.join(dstdir, os.path.basename(src))
@@ -110,7 +113,7 @@
110 self.files = []113 self.files = []
111 self.channel_switch = None114 self.channel_switch = None
112 # Other public attributes.115 # Other public attributes.
113 self.downloader = DBusDownloadManager()116 self.downloader = get_download_manager()
114 self._next.append(self._cleanup)117 self._next.append(self._cleanup)
115118
116 def __iter__(self):119 def __iter__(self):
@@ -211,7 +214,7 @@
211 # I think it makes no sense to check the blacklist when we're214 # I think it makes no sense to check the blacklist when we're
212 # downloading a blacklist file.215 # downloading a blacklist file.
213 log.info('Looking for blacklist: {}'.format(216 log.info('Looking for blacklist: {}'.format(
214 urljoin(config.service.https_base, url)))217 urljoin(config.https_base, url)))
215 get_keyring('blacklist', url, 'image-master')218 get_keyring('blacklist', url, 'image-master')
216 except SignatureError:219 except SignatureError:
217 log.exception('No signed blacklist found')220 log.exception('No signed blacklist found')
@@ -247,7 +250,7 @@
247 url = 'gpg/blacklist.tar.xz'250 url = 'gpg/blacklist.tar.xz'
248 try:251 try:
249 log.info('Looking for blacklist again: {}',252 log.info('Looking for blacklist again: {}',
250 urljoin(config.service.https_base, url))253 urljoin(config.https_base, url))
251 get_keyring('blacklist', url, 'image-master')254 get_keyring('blacklist', url, 'image-master')
252 except FileNotFoundError:255 except FileNotFoundError:
253 log.info('No blacklist found on second attempt')256 log.info('No blacklist found on second attempt')
@@ -273,9 +276,9 @@
273 get_keyring(276 get_keyring(
274 'image-signing', 'gpg/image-signing.tar.xz', 'image-master',277 'image-signing', 'gpg/image-signing.tar.xz', 'image-master',
275 self.blacklist)278 self.blacklist)
276 channels_url = urljoin(config.service.https_base, 'channels.json')279 channels_url = urljoin(config.https_base, 'channels.json')
277 channels_path = os.path.join(config.tempdir, 'channels.json')280 channels_path = os.path.join(config.tempdir, 'channels.json')
278 asc_url = urljoin(config.service.https_base, 'channels.json.asc')281 asc_url = urljoin(config.https_base, 'channels.json.asc')
279 asc_path = os.path.join(config.tempdir, 'channels.json.asc')282 asc_path = os.path.join(config.tempdir, 'channels.json.asc')
280 log.info('Looking for: {}', channels_url)283 log.info('Looking for: {}', channels_url)
281 with ExitStack() as stack:284 with ExitStack() as stack:
@@ -329,8 +332,8 @@
329 self._next.append(partial(self._get_index, device.index))332 self._next.append(partial(self._get_index, device.index))
330333
331 def _get_device_keyring(self, keyring):334 def _get_device_keyring(self, keyring):
332 keyring_url = urljoin(config.service.https_base, keyring.path)335 keyring_url = urljoin(config.https_base, keyring.path)
333 asc_url = urljoin(config.service.https_base, keyring.signature)336 asc_url = urljoin(config.https_base, keyring.signature)
334 log.info('getting device keyring: {}', keyring_url)337 log.info('getting device keyring: {}', keyring_url)
335 get_keyring(338 get_keyring(
336 'device-signing', (keyring_url, asc_url), 'image-signing',339 'device-signing', (keyring_url, asc_url), 'image-signing',
@@ -378,7 +381,7 @@
378381
379 def _get_index(self, index):382 def _get_index(self, index):
380 """Get and verify the index.json file."""383 """Get and verify the index.json file."""
381 index_url = urljoin(config.service.https_base, index)384 index_url = urljoin(config.https_base, index)
382 asc_url = index_url + '.asc'385 asc_url = index_url + '.asc'
383 index_path = os.path.join(config.tempdir, 'index.json')386 index_path = os.path.join(config.tempdir, 'index.json')
384 asc_path = index_path + '.asc'387 asc_path = index_path + '.asc'
@@ -410,7 +413,7 @@
410 # winner. Otherwise, trust the configured build number.413 # winner. Otherwise, trust the configured build number.
411 channel = self.channels[config.channel]414 channel = self.channels[config.channel]
412 # channel_target is the channel we're on based on the alias mapping in415 # channel_target is the channel we're on based on the alias mapping in
413 # our channel.ini file. channel_alias is the alias mapping in the416 # our config files. channel_alias is the alias mapping in the
414 # channel.json file, i.e. the channel an update will put us on.417 # channel.json file, i.e. the channel an update will put us on.
415 channel_target = getattr(config.service, 'channel_target', None)418 channel_target = getattr(config.service, 'channel_target', None)
416 channel_alias = getattr(channel, 'alias', None)419 channel_alias = getattr(channel, 'alias', None)
@@ -418,17 +421,23 @@
418 channel_target is None or421 channel_target is None or
419 channel_alias == channel_target):422 channel_alias == channel_target):
420 build_number = config.build_number423 build_number = config.build_number
421 elif config.build_number_cli is not None:
422 # An explicit --build on the command line still takes precedence.
423 build_number = config.build_number_cli
424 else:424 else:
425 # This is a channel switch caused by a new alias.425 # This is a channel switch caused by a new alias. Unless the
426 build_number = 0426 # build number has been explicitly overridden on the command line
427 # via --build/-b, use build number 0 to force a full update.
428 build_number = (config.build_number
429 if config.build_number_override
430 else 0)
427 self.channel_switch = (channel_target, channel_alias)431 self.channel_switch = (channel_target, channel_alias)
428 candidates = get_candidates(self.index, build_number)432 candidates = get_candidates(self.index, build_number)
433 log.debug('Candidates from build# {}: {}'.format(
434 build_number, len(candidates)))
429 if self._filter is not None:435 if self._filter is not None:
430 candidates = self._filter(candidates)436 candidates = self._filter(candidates)
431 self.winner = config.hooks.scorer().choose(candidates)437 self.winner = config.hooks.scorer().choose(
438 candidates, (channel_target
439 if channel_alias is None
440 else channel_alias))
432 # If there is no winning upgrade candidate, then there's nothing more441 # If there is no winning upgrade candidate, then there's nothing more
433 # to do. We can skip everything between downloading the files and442 # to do. We can skip everything between downloading the files and
434 # doing the reboot.443 # doing the reboot.
@@ -474,11 +483,11 @@
474 else:483 else:
475 # Add the data file, which has a checksum.484 # Add the data file, which has a checksum.
476 downloads.append(Record(485 downloads.append(Record(
477 urljoin(config.service.http_base, filerec.path),486 urljoin(config.http_base, filerec.path),
478 dst, checksum))487 dst, checksum))
479 # Add the signature file, which does not have a checksum.488 # Add the signature file, which does not have a checksum.
480 downloads.append(Record(489 downloads.append(Record(
481 urljoin(config.service.http_base, filerec.signature),490 urljoin(config.http_base, filerec.signature),
482 asc))491 asc))
483 signatures.append((dst, asc))492 signatures.append((dst, asc))
484 checksums.append((dst, checksum))493 checksums.append((dst, checksum))
@@ -603,9 +612,9 @@
603 file=fp)612 file=fp)
604 # The filesystem must be unmounted.613 # The filesystem must be unmounted.
605 print('unmount system', file=fp)614 print('unmount system', file=fp)
606 self._next.append(self._reboot)615 self._next.append(self._apply)
607616
608 def _reboot(self):617 def _apply(self):
609 log.info('rebooting')618 log.info('applying')
610 config.hooks.reboot().reboot()619 config.hooks.apply().apply()
611 # Nothing more to do.620 # Nothing more to do.
612621
=== modified file 'systemimage/testing/controller.py'
--- systemimage/testing/controller.py 2014-09-17 13:41:31 +0000
+++ systemimage/testing/controller.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -29,24 +29,31 @@
29import psutil29import psutil
30import subprocess30import subprocess
3131
32try:
33 import pycurl
34except ImportError:
35 pycurl = None
36
32from contextlib import ExitStack37from contextlib import ExitStack
33from distutils.spawn import find_executable38from distutils.spawn import find_executable
34from pkg_resources import resource_string as resource_bytes39from pkg_resources import resource_string as resource_bytes
35from systemimage.helpers import temporary_directory40from systemimage.helpers import temporary_directory
36from systemimage.testing.helpers import (41from systemimage.testing.helpers import (
37 data_path, find_dbus_process, reset_envar)42 data_path, find_dbus_process, makedirs, reset_envar, wait_for_service)
43from unittest.mock import patch
3844
3945
40SPACE = ' '46SPACE = ' '
41OVERRIDE = os.environ.get('SYSTEMIMAGE_DBUS_DAEMON_HUP_SLEEP_SECONDS')47DLSERVICE = os.environ.get(
42HUP_SLEEP = (0 if OVERRIDE is None else int(OVERRIDE))48 'SYSTEMIMAGE_DLSERVICE',
49 '/usr/bin/ubuntu-download-manager'
50 # For debugging the in-tree version of u-d-m.
51 #'/bin/sh $HOME/projects/phone/trunk/tools/runme.sh'
52 )
4353
4454
45def start_system_image(controller):55def start_system_image(controller):
46 bus = dbus.SystemBus()56 wait_for_service(reload=False)
47 service = bus.get_object('com.canonical.SystemImage', '/Service')
48 iface = dbus.Interface(service, 'com.canonical.SystemImage')
49 iface.Info()
50 process = find_dbus_process(controller.ini_path)57 process = find_dbus_process(controller.ini_path)
51 if process is None:58 if process is None:
52 raise RuntimeError('Could not start system-image-dbus')59 raise RuntimeError('Could not start system-image-dbus')
@@ -78,12 +85,14 @@
7885
7986
80def start_downloader(controller):87def start_downloader(controller):
81 bus = dbus.SystemBus()88 service = dbus.SystemBus().get_object('org.freedesktop.DBus', '/')
82 service = bus.get_object('com.canonical.applications.Downloader', '/')89 iface = dbus.Interface(service, 'org.freedesktop.DBus')
83 iface = dbus.Interface(90 reply = 0
84 service, 'com.canonical.applications.DownloadManager')91 while reply != 2:
92 reply = iface.StartServiceByName(
93 'com.canonical.applications.Downloader', 0)
94 time.sleep(0.1)
85 # Something innocuous.95 # Something innocuous.
86 iface.defaultThrottle()
87 process = _find_udm_process()96 process = _find_udm_process()
88 if process is None:97 if process is None:
89 raise RuntimeError('Could not start ubuntu-download-manager')98 raise RuntimeError('Could not start ubuntu-download-manager')
@@ -105,24 +114,30 @@
105 process.wait(60)114 process.wait(60)
106115
107116
108DLSERVICE = '/usr/bin/ubuntu-download-manager'
109# For debugging the in-tree version of u-d-m.
110#DLSERVICE = '/bin/sh /home/barry/projects/phone/runme'
111
112
113SERVICES = [117SERVICES = [
114 ('com.canonical.SystemImage',118 ('com.canonical.SystemImage',
115 '{python} -m {self.MODULE} -C {self.ini_path} --testing {self.mode}',119 '{python} -m {self.MODULE} -C {self.ini_path} '
120 '{self.curl_cert} --testing {self.mode}',
116 start_system_image,121 start_system_image,
117 stop_system_image,122 stop_system_image,
118 ),123 ),
119 ('com.canonical.applications.Downloader',124 ]
125
126
127if pycurl is None:
128 USING_PYCURL = False
129else:
130 USING_PYCURL = int(os.environ.get('SYSTEMIMAGE_PYCURL', '0'))
131
132if not USING_PYCURL:
133 SERVICES.append(
134 ('com.canonical.applications.Downloader',
120 DLSERVICE +135 DLSERVICE +
121 ' {self.certs} -disable-timeout -stoppable -log-dir {self.tmpdir}',136 ' {self.udm_certs} -disable-timeout -stoppable -log-dir {self.tmpdir}',
122 start_downloader,137 start_downloader,
123 stop_downloader,138 stop_downloader,
124 ),139 )
125 ]140 )
126141
127142
128class Controller:143class Controller:
@@ -131,17 +146,19 @@
131 MODULE = 'systemimage.testing.service'146 MODULE = 'systemimage.testing.service'
132147
133 def __init__(self, logfile=None, loglevel='info'):148 def __init__(self, logfile=None, loglevel='info'):
149 self.loglevel = loglevel
134 # Non-public.150 # Non-public.
135 self._stack = ExitStack()151 self._stack = ExitStack()
136 self._stoppers = []152 self._stoppers = []
137 # Public.153 # Public.
138 self.tmpdir = self._stack.enter_context(temporary_directory())154 self.tmpdir = self._stack.enter_context(temporary_directory())
139 self.config_path = os.path.join(self.tmpdir, 'dbus-system.conf')155 self.config_path = os.path.join(self.tmpdir, 'dbus-system.conf')
140 self.ini_path = None
141 self.serverdir = self._stack.enter_context(temporary_directory())156 self.serverdir = self._stack.enter_context(temporary_directory())
142 self.daemon_pid = None157 self.daemon_pid = None
143 self.mode = 'live'158 self.mode = 'live'
144 self.certs = ''159 self.udm_certs = ''
160 self.curl_cert = ''
161 self.patcher = None
145 # Set up the dbus-daemon system configuration file.162 # Set up the dbus-daemon system configuration file.
146 path = data_path('dbus-system.conf.in')163 path = data_path('dbus-system.conf.in')
147 with open(path, 'r', encoding='utf-8') as fp:164 with open(path, 'r', encoding='utf-8') as fp:
@@ -151,19 +168,27 @@
151 with open(self.config_path, 'w', encoding='utf-8') as fp:168 with open(self.config_path, 'w', encoding='utf-8') as fp:
152 fp.write(config)169 fp.write(config)
153 # We need a client.ini file for the subprocess.170 # We need a client.ini file for the subprocess.
154 ini_tmpdir = self._stack.enter_context(temporary_directory())171 self.ini_tmpdir = self._stack.enter_context(temporary_directory())
155 ini_vardir = self._stack.enter_context(temporary_directory())172 self.ini_vardir = self._stack.enter_context(temporary_directory())
156 ini_logfile = (os.path.join(ini_tmpdir, 'client.log')173 self.ini_logfile = (os.path.join(self.ini_tmpdir, 'client.log')
157 if logfile is None174 if logfile is None
158 else logfile)175 else logfile)
159 self.ini_path = os.path.join(self.tmpdir, 'client.ini')176 self.ini_path = os.path.join(self.tmpdir, 'config.d')
177 makedirs(self.ini_path)
178 self._reset_configs()
179
180 def _reset_configs(self):
181 for filename in os.listdir(self.ini_path):
182 if filename.endswith('.ini'):
183 os.remove(os.path.join(self.ini_path, filename))
160 template = resource_bytes(184 template = resource_bytes(
161 'systemimage.tests.data', 'config_03.ini').decode('utf-8')185 'systemimage.tests.data', '01.ini').decode('utf-8')
162 with open(self.ini_path, 'w', encoding='utf-8') as fp:186 defaults = os.path.join(self.ini_path, '00_defaults.ini')
163 print(template.format(tmpdir=ini_tmpdir,187 with open(defaults, 'w', encoding='utf-8') as fp:
164 vardir=ini_vardir,188 print(template.format(tmpdir=self.ini_tmpdir,
165 logfile=ini_logfile,189 vardir=self.ini_vardir,
166 loglevel=loglevel),190 logfile=self.ini_logfile,
191 loglevel=self.loglevel),
167 file=fp)192 file=fp)
168193
169 def _configure_services(self):194 def _configure_services(self):
@@ -184,16 +209,41 @@
184 self._stoppers.append(stopper)209 self._stoppers.append(stopper)
185 # If the dbus-daemon is running, reload its configuration files.210 # If the dbus-daemon is running, reload its configuration files.
186 if self.daemon_pid is not None:211 if self.daemon_pid is not None:
187 service = dbus.SystemBus().get_object('org.freedesktop.DBus', '/')212 wait_for_service()
188 iface = dbus.Interface(service, 'org.freedesktop.DBus')213
189 iface.ReloadConfig()214 def _set_udm_certs(self, cert_pem, certificate_path):
190 time.sleep(HUP_SLEEP)215 self.udm_certs = (
216 '' if cert_pem is None
217 else '-self-signed-certs ' + certificate_path)
218
219 def _set_curl_certs(self, cert_pem, certificate_path):
220 # We have to set up the PyCURL downloader's self-signed certificate for
221 # the test in two ways. First, because we might be spawning the D-Bus
222 # service, we have to pass the path to the cert to that service...
223 self.curl_cert = (
224 '' if cert_pem is None
225 else '--self-signed-cert ' + certificate_path)
226 # ...but the controller is also used to set the mode for foreground
227 # tests, such as test_download.py. Here we don't spawn any D-Bus
228 # processes, but we still have to mock make_testable() in curl.py so
229 # that the PyCURL object accepts the self-signed cert.
230 if self.patcher is not None:
231 self.patcher.stop()
232 self.patcher = None
233 if cert_pem is not None:
234 def self_sign(c):
235 c.setopt(pycurl.CAINFO, certificate_path)
236 self.patcher = patch('systemimage.curl.make_testable', self_sign)
237 self.patcher.start()
191238
192 def set_mode(self, *, cert_pem=None, service_mode=''):239 def set_mode(self, *, cert_pem=None, service_mode=''):
193 self.mode = service_mode240 self.mode = service_mode
194 self.certs = (241 certificate_path = data_path(cert_pem)
195 '' if cert_pem is None242 if USING_PYCURL:
196 else '-self-signed-certs ' + data_path(cert_pem))243 self._set_curl_certs(cert_pem, certificate_path)
244 else:
245 self._set_udm_certs(cert_pem, certificate_path)
246 self._reset_configs()
197 self._configure_services()247 self._configure_services()
198248
199 def _start(self):249 def _start(self):
@@ -213,7 +263,7 @@
213 daemon_exe,263 daemon_exe,
214 #'/usr/lib/x86_64-linux-gnu/dbus-1.0/debug-build/bin/dbus-daemon',264 #'/usr/lib/x86_64-linux-gnu/dbus-1.0/debug-build/bin/dbus-daemon',
215 '--fork',265 '--fork',
216 '--config-file=' + self.config_path,266 '--config-file=' + str(self.config_path),
217 # Return the address and pid on stdout.267 # Return the address and pid on stdout.
218 '--print-address=1',268 '--print-address=1',
219 '--print-pid=1',269 '--print-pid=1',
220270
=== modified file 'systemimage/testing/dbus.py'
--- systemimage/testing/dbus.py 2014-09-26 14:36:34 +0000
+++ systemimage/testing/dbus.py 2015-05-20 14:55:53 +0000
@@ -1,4 +1,4 @@
1# Copyright (C) 2013-2014 Canonical Ltd.1# Copyright (C) 2013-2015 Canonical Ltd.
2# Author: Barry Warsaw <barry@ubuntu.com>2# Author: Barry Warsaw <barry@ubuntu.com>
33
4# This program is free software: you can redistribute it and/or modify4# This program is free software: you can redistribute it and/or modify
@@ -23,6 +23,11 @@
2323
24import os24import os
2525
26try:
27 import pycurl
28except ImportError:
29 pycurl = None
30
26from dbus.service import method, signal31from dbus.service import method, signal
27from gi.repository import GLib32from gi.repository import GLib
28from systemimage.api import Mediator33from systemimage.api import Mediator
@@ -45,7 +50,7 @@
45 fp.write(SPACE.join(args[0]).strip())50 fp.write(SPACE.join(args[0]).strip())
4651
4752
48def instrument(config, stack):53def instrument(config, stack, cert_file):
49 """Instrument the system for testing."""54 """Instrument the system for testing."""
50 # Ensure the destination directories exist.55 # Ensure the destination directories exist.
51 makedirs(config.updater.data_partition)56 makedirs(config.updater.data_partition)
@@ -54,9 +59,16 @@
54 # file which the testing parent process can open and read.59 # file which the testing parent process can open and read.
55 safe_reboot = _ActionLog('reboot.log')60 safe_reboot = _ActionLog('reboot.log')
56 stack.enter_context(61 stack.enter_context(
57 patch('systemimage.reboot.check_call', safe_reboot.write))62 patch('systemimage.apply.check_call', safe_reboot.write))
58 stack.enter_context(63 stack.enter_context(
59 patch('systemimage.device.check_output', return_value='nexus7'))64 patch('systemimage.device.check_output', return_value='nexus7'))
65 # If available, patch the PyCURL downloader to accept self-signed
66 # certificates.
67 if pycurl is not None:
68 def self_sign(c):
69 c.setopt(pycurl.CAINFO, cert_file)
70 stack.enter_context(
71 patch('systemimage.curl.make_testable', self_sign))
6072
6173
62class _LiveTestableService(Service):74class _LiveTestableService(Service):
@@ -65,6 +77,7 @@
65 @log_and_exit77 @log_and_exit
66 @method('com.canonical.SystemImage')78 @method('com.canonical.SystemImage')
67 def Reset(self):79 def Reset(self):
80 config.reload()
68 self._api = Mediator()81 self._api = Mediator()
69 try:82 try:
70 self._checking.release()83 self._checking.release()
@@ -72,7 +85,6 @@
72 # Lock is already released.85 # Lock is already released.
73 pass86 pass
74 self._update = None87 self._update = None
75 self._downloading = False
76 self._rebootable = False88 self._rebootable = False
77 self._failure_count = 089 self._failure_count = 0
78 del config.build_number90 del config.build_number
@@ -189,9 +201,9 @@
189 @method('com.canonical.SystemImage')201 @method('com.canonical.SystemImage')
190 def ApplyUpdate(self):202 def ApplyUpdate(self):
191 # Always succeeds.203 # Always succeeds.
192 def _rebooting():204 def _applied():
193 self.Rebooting(True)205 self.Applied(True)
194 GLib.timeout_add(50, _rebooting)206 GLib.timeout_add(50, _applied)
195207
196208
197class _UpdateManualSuccess(_UpdateAutoSuccess):209class _UpdateManualSuccess(_UpdateAutoSuccess):
@@ -259,9 +271,9 @@
259 @method('com.canonical.SystemImage')271 @method('com.canonical.SystemImage')
260 def ApplyUpdate(self):272 def ApplyUpdate(self):