Merge lp:~tyler-baker/lava-dispatcher/consolidate-busybox-httpd into lp:lava-dispatcher
- consolidate-busybox-httpd
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 677 |
Proposed branch: | lp:~tyler-baker/lava-dispatcher/consolidate-busybox-httpd |
Merge into: | lp:lava-dispatcher |
Diff against target: |
261 lines (+49/-108) 3 files modified
lava_dispatcher/device/bootloader.py (+6/-49) lava_dispatcher/device/ipmi_pxe.py (+4/-55) lava_dispatcher/device/target.py (+39/-4) |
To merge this branch: | bzr merge lp:~tyler-baker/lava-dispatcher/consolidate-busybox-httpd |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Antonio Terceiro | Approve | ||
Review via email: mp+184403@code.launchpad.net |
Commit message
Description of the change
Consolidation of the busybox httpd logic. The ipmi_pxe and bootloader class both use these functions to allow the exchange of files to and from the target. It makes sense to reuse a single implementation, rather than maintaining both.
* Moved all the functionality into target.py for reuse.
* start_http_server -> start_busybox_
* stop_http_server -> stop_busybox_
Tested these changes on a bootloader device.
http://
However, I cannot test these changes on a highbank node until merged into staging.
Michael Hudson-Doyle (mwhudson) wrote : | # |
Antonio Terceiro (terceiro) wrote : | # |
Looks good to me.
I also agree with Michael's comments - doing that will simplify the code even more.
review approve
Tyler Baker (tyler-baker) wrote : | # |
I'll update the branch per Michael's comments after I retest. Thanks for the suggestion!
- 676. By Tyler Baker
-
Remove httpd pid
- 677. By Tyler Baker
-
Remove the dead code
Tyler Baker (tyler-baker) wrote : | # |
Branch updated to reflect comments.
Tested the changes on a bootloader device:
http://
Merging to test on the highbank in staging.
Tyler Baker (tyler-baker) wrote : | # |
Confirmed logic works correctly on highbank in staging.
Preview Diff
1 | === modified file 'lava_dispatcher/device/bootloader.py' | |||
2 | --- lava_dispatcher/device/bootloader.py 2013-08-30 22:15:05 +0000 | |||
3 | +++ lava_dispatcher/device/bootloader.py 2013-09-10 16:31:28 +0000 | |||
4 | @@ -51,7 +51,6 @@ | |||
5 | 51 | self._boot_cmds = None | 51 | self._boot_cmds = None |
6 | 52 | self._lava_cmds = None | 52 | self._lava_cmds = None |
7 | 53 | self._uboot_boot = False | 53 | self._uboot_boot = False |
8 | 54 | self._http_pid = None | ||
9 | 55 | # This is the offset into the path, used to reference bootfiles | 54 | # This is the offset into the path, used to reference bootfiles |
10 | 56 | self._offset = self.scratch_dir.index('images') | 55 | self._offset = self.scratch_dir.index('images') |
11 | 57 | 56 | ||
12 | @@ -115,11 +114,13 @@ | |||
13 | 115 | 114 | ||
14 | 116 | def deploy_linaro(self, hwpack, rfs, bootloadertype): | 115 | def deploy_linaro(self, hwpack, rfs, bootloadertype): |
15 | 117 | self._uboot_boot = False | 116 | self._uboot_boot = False |
17 | 118 | super(BootloaderTarget, self).deploy_linaro(hwpack, rfs, bootloadertype) | 117 | super(BootloaderTarget, self).deploy_linaro(hwpack, rfs, |
18 | 118 | bootloadertype) | ||
19 | 119 | 119 | ||
20 | 120 | def deploy_linaro_prebuilt(self, image, bootloadertype): | 120 | def deploy_linaro_prebuilt(self, image, bootloadertype): |
21 | 121 | self._uboot_boot = False | 121 | self._uboot_boot = False |
23 | 122 | super(BootloaderTarget, self).deploy_linaro_prebuilt(image, bootloadertype) | 122 | super(BootloaderTarget, self).deploy_linaro_prebuilt(image, |
24 | 123 | bootloadertype) | ||
25 | 123 | 124 | ||
26 | 124 | def _inject_boot_cmds(self): | 125 | def _inject_boot_cmds(self): |
27 | 125 | if self._is_job_defined_boot_cmds(self.config.boot_cmds): | 126 | if self._is_job_defined_boot_cmds(self.config.boot_cmds): |
28 | @@ -164,28 +165,6 @@ | |||
29 | 164 | else: | 165 | else: |
30 | 165 | super(BootloaderTarget, self)._boot_linaro_image() | 166 | super(BootloaderTarget, self)._boot_linaro_image() |
31 | 166 | 167 | ||
32 | 167 | def start_http_server(self, runner, ip): | ||
33 | 168 | if self._http_pid is not None: | ||
34 | 169 | raise OperationFailed("busybox httpd already running with pid %d" | ||
35 | 170 | % self._http_pid) | ||
36 | 171 | # busybox produces no output to parse for, | ||
37 | 172 | # so run it in the bg and get its pid | ||
38 | 173 | runner.run('busybox httpd -f &') | ||
39 | 174 | runner.run('echo pid:$!:pid', response="pid:(\d+):pid", timeout=10) | ||
40 | 175 | if runner.match_id != 0: | ||
41 | 176 | raise OperationFailed("busybox httpd did not start") | ||
42 | 177 | else: | ||
43 | 178 | self._http_pid = runner.match.group(1) | ||
44 | 179 | url_base = "http://%s" % ip | ||
45 | 180 | return url_base | ||
46 | 181 | |||
47 | 182 | def stop_http_server(self, runner): | ||
48 | 183 | if self._http_pid is None: | ||
49 | 184 | raise OperationFailed("busybox httpd not running, \ | ||
50 | 185 | but stop_http_server called.") | ||
51 | 186 | runner.run('kill %s' % self._http_pid) | ||
52 | 187 | self._http_pid = None | ||
53 | 188 | |||
54 | 189 | @contextlib.contextmanager | 168 | @contextlib.contextmanager |
55 | 190 | def file_system(self, partition, directory): | 169 | def file_system(self, partition, directory): |
56 | 191 | if self._uboot_boot: | 170 | if self._uboot_boot: |
57 | @@ -202,7 +181,7 @@ | |||
58 | 202 | runner.run('cd /tmp') # need to be in same dir as fs.tgz | 181 | runner.run('cd /tmp') # need to be in same dir as fs.tgz |
59 | 203 | 182 | ||
60 | 204 | ip = runner.get_target_ip() | 183 | ip = runner.get_target_ip() |
62 | 205 | url_base = self.start_http_server(runner, ip) | 184 | url_base = self._start_busybox_http_server(runner, ip) |
63 | 206 | 185 | ||
64 | 207 | url = url_base + '/fs.tgz' | 186 | url = url_base + '/fs.tgz' |
65 | 208 | logging.info("Fetching url: %s" % url) | 187 | logging.info("Fetching url: %s" % url) |
66 | @@ -226,32 +205,10 @@ | |||
67 | 226 | runner.run('rm -rf %s' % targetdir) | 205 | runner.run('rm -rf %s' % targetdir) |
68 | 227 | self._target_extract(runner, tf, parent_dir) | 206 | self._target_extract(runner, tf, parent_dir) |
69 | 228 | finally: | 207 | finally: |
71 | 229 | self.stop_http_server(runner) | 208 | self._stop_busybox_http_server(runner) |
72 | 230 | else: | 209 | else: |
73 | 231 | with super(BootloaderTarget, self).file_system( | 210 | with super(BootloaderTarget, self).file_system( |
74 | 232 | partition, directory) as path: | 211 | partition, directory) as path: |
75 | 233 | yield path | 212 | yield path |
76 | 234 | 213 | ||
77 | 235 | def _target_extract(self, runner, tar_file, dest, timeout=-1): | ||
78 | 236 | tmpdir = self.context.config.lava_image_tmpdir | ||
79 | 237 | url = self.context.config.lava_image_url | ||
80 | 238 | tar_file = tar_file.replace(tmpdir, '') | ||
81 | 239 | tar_url = '/'.join(u.strip('/') for u in [url, tar_file]) | ||
82 | 240 | self._target_extract_url(runner, tar_url, dest, timeout=timeout) | ||
83 | 241 | |||
84 | 242 | def _target_extract_url(self, runner, tar_url, dest, timeout=-1): | ||
85 | 243 | decompression_cmd = '' | ||
86 | 244 | if tar_url.endswith('.gz') or tar_url.endswith('.tgz'): | ||
87 | 245 | decompression_cmd = '| /bin/gzip -dc' | ||
88 | 246 | elif tar_url.endswith('.bz2'): | ||
89 | 247 | decompression_cmd = '| /bin/bzip2 -dc' | ||
90 | 248 | elif tar_url.endswith('.tar'): | ||
91 | 249 | decompression_cmd = '' | ||
92 | 250 | else: | ||
93 | 251 | raise RuntimeError('bad file extension: %s' % tar_url) | ||
94 | 252 | |||
95 | 253 | runner.run('wget -O - %s %s | /bin/tar -C %s -xmf -' | ||
96 | 254 | % (tar_url, decompression_cmd, dest), | ||
97 | 255 | timeout=timeout) | ||
98 | 256 | |||
99 | 257 | target_class = BootloaderTarget | 214 | target_class = BootloaderTarget |
100 | 258 | 215 | ||
101 | === modified file 'lava_dispatcher/device/ipmi_pxe.py' | |||
102 | --- lava_dispatcher/device/ipmi_pxe.py 2013-08-30 22:15:05 +0000 | |||
103 | +++ lava_dispatcher/device/ipmi_pxe.py 2013-09-10 16:31:28 +0000 | |||
104 | @@ -32,7 +32,6 @@ | |||
105 | 32 | ) | 32 | ) |
106 | 33 | from lava_dispatcher.errors import ( | 33 | from lava_dispatcher.errors import ( |
107 | 34 | CriticalError, | 34 | CriticalError, |
108 | 35 | OperationFailed, | ||
109 | 36 | ) | 35 | ) |
110 | 37 | from lava_dispatcher.downloader import ( | 36 | from lava_dispatcher.downloader import ( |
111 | 38 | download_image, | 37 | download_image, |
112 | @@ -176,7 +175,8 @@ | |||
113 | 176 | runner.run('/bin/tar -cmzf /tmp/fs.tgz -C %s %s' % (parent_dir, target_name)) | 175 | runner.run('/bin/tar -cmzf /tmp/fs.tgz -C %s %s' % (parent_dir, target_name)) |
114 | 177 | runner.run('cd /tmp') # need to be in same dir as fs.tgz | 176 | runner.run('cd /tmp') # need to be in same dir as fs.tgz |
115 | 178 | 177 | ||
117 | 179 | url_base = runner.start_http_server() | 178 | ip = runner.get_target_ip() |
118 | 179 | url_base = self._start_busybox_http_server(runner, ip) | ||
119 | 180 | 180 | ||
120 | 181 | url = url_base + '/fs.tgz' | 181 | url = url_base + '/fs.tgz' |
121 | 182 | logging.info("Fetching url: %s" % url) | 182 | logging.info("Fetching url: %s" % url) |
122 | @@ -200,38 +200,16 @@ | |||
123 | 200 | self._target_extract(runner, tf, parent_dir) | 200 | self._target_extract(runner, tf, parent_dir) |
124 | 201 | 201 | ||
125 | 202 | finally: | 202 | finally: |
127 | 203 | runner.stop_http_server() | 203 | self._stop_busybox_http_server() |
128 | 204 | runner.run('umount /mnt') | 204 | runner.run('umount /mnt') |
129 | 205 | 205 | ||
130 | 206 | def _target_extract(self, runner, tar_file, dest, timeout=-1): | ||
131 | 207 | tmpdir = self.context.config.lava_image_tmpdir | ||
132 | 208 | url = self.context.config.lava_image_url | ||
133 | 209 | tar_file = tar_file.replace(tmpdir, '') | ||
134 | 210 | tar_url = '/'.join(u.strip('/') for u in [url, tar_file]) | ||
135 | 211 | self._target_extract_url(runner, tar_url, dest, timeout=timeout) | ||
136 | 212 | |||
137 | 213 | def _target_extract_url(self, runner, tar_url, dest, timeout=-1): | ||
138 | 214 | decompression_cmd = '' | ||
139 | 215 | if tar_url.endswith('.gz') or tar_url.endswith('.tgz'): | ||
140 | 216 | decompression_cmd = '| /bin/gzip -dc' | ||
141 | 217 | elif tar_url.endswith('.bz2'): | ||
142 | 218 | decompression_cmd = '| /bin/bzip2 -dc' | ||
143 | 219 | elif tar_url.endswith('.tar'): | ||
144 | 220 | decompression_cmd = '' | ||
145 | 221 | else: | ||
146 | 222 | raise RuntimeError('bad file extension: %s' % tar_url) | ||
147 | 223 | |||
148 | 224 | runner.run('wget -O - %s %s | /bin/tar -C %s -xmf -' | ||
149 | 225 | % (tar_url, decompression_cmd, dest), | ||
150 | 226 | timeout=timeout) | ||
151 | 227 | |||
152 | 228 | @contextlib.contextmanager | 206 | @contextlib.contextmanager |
153 | 229 | def _as_master(self): | 207 | def _as_master(self): |
154 | 230 | self.bootcontrol.power_on_boot_master() | 208 | self.bootcontrol.power_on_boot_master() |
155 | 231 | self.proc.expect("\(initramfs\)") | 209 | self.proc.expect("\(initramfs\)") |
156 | 232 | self.proc.sendline('export PS1="%s"' % self.MASTER_PS1) | 210 | self.proc.sendline('export PS1="%s"' % self.MASTER_PS1) |
157 | 233 | self.proc.expect(self.MASTER_PS1_PATTERN, timeout=180, lava_no_logging=1) | 211 | self.proc.expect(self.MASTER_PS1_PATTERN, timeout=180, lava_no_logging=1) |
159 | 234 | runner = BusyboxHttpdMasterCommandRunner(self) | 212 | runner = MasterCommandRunner(self) |
160 | 235 | 213 | ||
161 | 236 | runner.run(". /scripts/functions") | 214 | runner.run(". /scripts/functions") |
162 | 237 | runner.run("DEVICE=%s configure_networking" % | 215 | runner.run("DEVICE=%s configure_networking" % |
163 | @@ -257,32 +235,3 @@ | |||
164 | 257 | 235 | ||
165 | 258 | 236 | ||
166 | 259 | target_class = IpmiPxeTarget | 237 | target_class = IpmiPxeTarget |
167 | 260 | |||
168 | 261 | |||
169 | 262 | class BusyboxHttpdMasterCommandRunner(MasterCommandRunner): | ||
170 | 263 | """A CommandRunner to use when the target is booted into the master image. | ||
171 | 264 | """ | ||
172 | 265 | http_pid = None | ||
173 | 266 | |||
174 | 267 | def __init__(self, target): | ||
175 | 268 | super(BusyboxHttpdMasterCommandRunner, self).__init__(target) | ||
176 | 269 | |||
177 | 270 | def start_http_server(self): | ||
178 | 271 | master_ip = self.get_master_ip() | ||
179 | 272 | if self.http_pid is not None: | ||
180 | 273 | raise OperationFailed("busybox httpd already running with pid %d" % self.http_pid) | ||
181 | 274 | # busybox produces no output to parse for, so run it in the bg and get its pid | ||
182 | 275 | self.run('busybox httpd -f &') | ||
183 | 276 | self.run('echo pid:$!:pid', response="pid:(\d+):pid", timeout=10) | ||
184 | 277 | if self.match_id != 0: | ||
185 | 278 | raise OperationFailed("busybox httpd did not start") | ||
186 | 279 | else: | ||
187 | 280 | self.http_pid = self.match.group(1) | ||
188 | 281 | url_base = "http://%s" % master_ip | ||
189 | 282 | return url_base | ||
190 | 283 | |||
191 | 284 | def stop_http_server(self): | ||
192 | 285 | if self.http_pid is None: | ||
193 | 286 | raise OperationFailed("busybox httpd not running, but stop_http_server called.") | ||
194 | 287 | self.run('kill %s' % self.http_pid) | ||
195 | 288 | self.http_pid = None | ||
196 | 289 | 238 | ||
197 | === modified file 'lava_dispatcher/device/target.py' | |||
198 | --- lava_dispatcher/device/target.py 2013-08-30 22:15:05 +0000 | |||
199 | +++ lava_dispatcher/device/target.py 2013-09-10 16:31:28 +0000 | |||
200 | @@ -191,13 +191,15 @@ | |||
201 | 191 | 191 | ||
202 | 192 | def _customize_bootloader(self, connection, boot_cmds): | 192 | def _customize_bootloader(self, connection, boot_cmds): |
203 | 193 | for line in boot_cmds: | 193 | for line in boot_cmds: |
205 | 194 | parts = re.match('^(?P<action>sendline|expect)\s*(?P<command>.*)', line) | 194 | parts = re.match('^(?P<action>sendline|expect)\s*(?P<command>.*)', |
206 | 195 | line) | ||
207 | 195 | if parts: | 196 | if parts: |
208 | 196 | try: | 197 | try: |
209 | 197 | action = parts.group('action') | 198 | action = parts.group('action') |
210 | 198 | command = parts.group('command') | 199 | command = parts.group('command') |
211 | 199 | except AttributeError as e: | 200 | except AttributeError as e: |
213 | 200 | raise Exception("Badly formatted command in boot_cmds %s" % e) | 201 | raise Exception("Badly formatted command in \ |
214 | 202 | boot_cmds %s" % e) | ||
215 | 201 | if action == "sendline": | 203 | if action == "sendline": |
216 | 202 | connection.send(command) | 204 | connection.send(command) |
217 | 203 | connection.sendline('') | 205 | connection.sendline('') |
218 | @@ -205,8 +207,41 @@ | |||
219 | 205 | command = re.escape(command) | 207 | command = re.escape(command) |
220 | 206 | connection.expect(command, timeout=300) | 208 | connection.expect(command, timeout=300) |
221 | 207 | else: | 209 | else: |
224 | 208 | self._wait_for_prompt(connection, self.config.bootloader_prompt, timeout=300) | 210 | self._wait_for_prompt(connection, |
225 | 209 | connection.sendline(line) | 211 | self.config.bootloader_prompt, |
226 | 212 | timeout=300) | ||
227 | 213 | connection.sendline(line) | ||
228 | 214 | |||
229 | 215 | def _target_extract(self, runner, tar_file, dest, timeout=-1): | ||
230 | 216 | tmpdir = self.context.config.lava_image_tmpdir | ||
231 | 217 | url = self.context.config.lava_image_url | ||
232 | 218 | tar_file = tar_file.replace(tmpdir, '') | ||
233 | 219 | tar_url = '/'.join(u.strip('/') for u in [url, tar_file]) | ||
234 | 220 | self._target_extract_url(runner, tar_url, dest, timeout=timeout) | ||
235 | 221 | |||
236 | 222 | def _target_extract_url(self, runner, tar_url, dest, timeout=-1): | ||
237 | 223 | decompression_cmd = '' | ||
238 | 224 | if tar_url.endswith('.gz') or tar_url.endswith('.tgz'): | ||
239 | 225 | decompression_cmd = '| /bin/gzip -dc' | ||
240 | 226 | elif tar_url.endswith('.bz2'): | ||
241 | 227 | decompression_cmd = '| /bin/bzip2 -dc' | ||
242 | 228 | elif tar_url.endswith('.tar'): | ||
243 | 229 | decompression_cmd = '' | ||
244 | 230 | else: | ||
245 | 231 | raise RuntimeError('bad file extension: %s' % tar_url) | ||
246 | 232 | |||
247 | 233 | runner.run('wget -O - %s %s | /bin/tar -C %s -xmf -' | ||
248 | 234 | % (tar_url, decompression_cmd, dest), | ||
249 | 235 | timeout=timeout) | ||
250 | 236 | |||
251 | 237 | def _start_busybox_http_server(self, runner, ip): | ||
252 | 238 | runner.run('busybox httpd -f &') | ||
253 | 239 | runner.run('echo $! > /tmp/httpd.pid') | ||
254 | 240 | url_base = "http://%s" % ip | ||
255 | 241 | return url_base | ||
256 | 242 | |||
257 | 243 | def _stop_busybox_http_server(self, runner): | ||
258 | 244 | runner.run('kill `cat /tmp/httpd.pid`') | ||
259 | 210 | 245 | ||
260 | 211 | def _customize_ubuntu(self, rootdir): | 246 | def _customize_ubuntu(self, rootdir): |
261 | 212 | self.deployment_data = Target.ubuntu_deployment_data | 247 | self.deployment_data = Target.ubuntu_deployment_data |
I know you're just moving code around so this is a side comment but does the httpd's pid really need to come to the host? Seems we could just execute
$ busybox httpd &
$ echo $! > /tmp/httpd.pid
to start and
$ kill `cat /tmp/httpd.pid`
to stop it.