1
=== added file '.flake8'
2
--- .flake8	1970-01-01 00:00:00 +0000
3
+++ .flake8	2017-07-14 19:31:31 +0000
4
@@ -0,0 +1,5 @@
5
1
[flake8]
6
2
ignore = E999, W503
7
3
exclude = venv_tests/
8
4
max-complexity = 7
9
5
max-line-length = 80
10
0
6
11
=== modified file '.project-flake8.sh'
12
--- .project-flake8.sh	2016-03-03 04:29:09 +0000
13
+++ .project-flake8.sh	2017-07-14 19:31:31 +0000
14
@@ -1,2 +1,20 @@
15
1
# Helpful doc: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
1
# Helpful doc: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
17
2
python3 -m flake8.run --ignore=W503 .
2
#!/bin/bash
18
3
RED='\033[0;31m'
19
4
GREEN='\033[0;32m'
20
5
21
6
OUTPUT=flake8_output.txt
22
7
if [ -f $OUTPUT ]; then
23
8
    rm $OUTPUT
24
9
fi
25
10
26
11
python3 -m flake8 --output-file=$OUTPUT .
27
12
28
13
result=$?
29
14
if [ $result != 0 ]; then
30
15
    echo -e -n "${RED}Flake8 errors. Check flake8 output for more information\n"
31
16
    cat $OUTPUT
32
17
    exit 1;
33
18
else
34
19
    echo -e -n "${GREEN}Congratulations!!! Static check PASSED\n"
35
20
fi
36
3
21
37
=== modified file 'debian/control'
38
--- debian/control	2017-02-03 20:16:09 +0000
39
+++ debian/control	2017-07-14 19:31:31 +0000
40
@@ -7,6 +7,9 @@
41
7
               python3-all-dev (>= 3.4),
7
               python3-all-dev (>= 3.4),
42
8
               python3-flake8,
8
               python3-flake8,
43
9
               python3-lxc,
9
               python3-lxc,
44
10
               python3-paramiko,
45
11
               python3-pexpect,
46
12
               python3-retrying,
47
10
               python3-setuptools,
13
               python3-setuptools,
48
11
               python3-yaml,
14
               python3-yaml,
49
12
Standards-Version: 3.9.3
15
Standards-Version: 3.9.3
50
@@ -22,7 +25,10 @@
51
22
         lxc-templates,
25
         lxc-templates,
52
23
         python3-yaml,
26
         python3-yaml,
53
24
         python3-lxc,
27
         python3-lxc,
54
28
         python3-paramiko,
55
29
         python3-pexpect,
56
25
         python3-pkg-resources,
30
         python3-pkg-resources,
57
31
         python3-retrying,
58
26
         android-tools-adb,
32
         android-tools-adb,
59
27
         phablet-tools,
33
         phablet-tools,
60
28
Description: Test release upgrades in a virtual environment
34
Description: Test release upgrades in a virtual environment
61
29
35
62
=== modified file 'upgrade_testing/command_line.py'
63
--- upgrade_testing/command_line.py	2017-05-23 09:54:22 +0000
64
+++ upgrade_testing/command_line.py	2017-07-14 19:31:31 +0000
65
@@ -69,6 +69,11 @@
66
69
    parser.add_argument(
69
    parser.add_argument(
67
70
        '--adt-args', '-a', default='',
70
        '--adt-args', '-a', default='',
68
71
        help='Arguments to pass through to the autopkgtest runner.')
71
        help='Arguments to pass through to the autopkgtest runner.')
69
72
    parser.add_argument(
70
73
        '--keep-overlay', '-k',
71
74
        default=False,
72
75
        action='store_true',
73
76
        help='Whether to keep the resulting overlay image')
74
72
    return parser.parse_args()
77
    return parser.parse_args()
75
73
78
76
74
79
77
@@ -137,7 +142,8 @@
78
137
    print('\n'.join(output))
142
    print('\n'.join(output))
79
138
143
80
139
144
82
140
def execute_adt_run(testsuite, testrun_files, output_dir, adt_args=''):
145
def execute_adt_run(testsuite, testrun_files, output_dir, adt_args='',
83
146
                    keep_overlay=False):
84
141
    """Prepare the autopkgtest to execute.
147
    """Prepare the autopkgtest to execute.
85
142
148
86
143
    Copy all the files into the expected place etc.
149
    Copy all the files into the expected place etc.
87
@@ -153,13 +159,14 @@
88
153
        output_dir,
159
        output_dir,
89
154
        testsuite.backend_args,
160
        testsuite.backend_args,
90
155
        adt_args,
161
        adt_args,
91
162
        keep_overlay
92
156
    )
163
    )
93
157
    subprocess.check_call(adt_run_command)
164
    subprocess.check_call(adt_run_command)
94
158
165
95
159
166
96
160
def get_adt_run_command(
167
def get_adt_run_command(
97
161
        provisioning, testrun_files, results_dir, backend_args=[],
168
        provisioning, testrun_files, results_dir, backend_args=[],
99
162
        adt_args=''):
169
        adt_args='', keep_overlay=False):
100
163
    """Construct the adt command to run.
170
    """Construct the adt command to run.
101
164
171
102
165
    :param provisioning: upgrade_testing.provisioning.ProvisionSpecification
172
    :param provisioning: upgrade_testing.provisioning.ProvisionSpecification
103
@@ -210,7 +217,8 @@
104
210
    )
217
    )
105
211
218
106
212
    backend_args = provisioning.get_adt_run_args(
219
    backend_args = provisioning.get_adt_run_args(
108
213
        tmp_dir=testrun_files.testrun_tmp_dir
220
        tmp_dir=testrun_files.testrun_tmp_dir,
109
221
        keep_overlay=keep_overlay
110
214
    ) + backend_args
222
    ) + backend_args
111
215
223
112
216
    return adt_cmd + ['---'] + backend_args
224
    return adt_cmd + ['---'] + backend_args
113
@@ -263,7 +271,9 @@
114
263
            output_dir = get_output_dir(args)
271
            output_dir = get_output_dir(args)
115
264
272
116
265
            execute_adt_run(testsuite, created_files, output_dir,
273
            execute_adt_run(testsuite, created_files, output_dir,
118
266
                            args.adt_args)
274
                            args.adt_args, args.keep_overlay)
119
275
120
276
            testsuite.provisioning.close()
121
267
277
122
268
        display_results(output_dir)
278
        display_results(output_dir)
123
269
279
124
270
280
125
=== modified file 'upgrade_testing/provisioning/_provisionconfig.py'
126
--- upgrade_testing/provisioning/_provisionconfig.py	2016-03-18 06:15:14 +0000
127
+++ upgrade_testing/provisioning/_provisionconfig.py	2017-07-14 19:31:31 +0000
128
@@ -58,6 +58,9 @@
129
58
        """Provision the stored backend."""
58
        """Provision the stored backend."""
130
59
        return self.backend.create(adt_base_path)
59
        return self.backend.create(adt_base_path)
131
60
60
132
61
    def close(self):
133
62
        return self.backend.close() if hasattr(self.backend, 'close') else None
134
63
135
61
    def get_adt_run_args(self, **kwargs):
64
    def get_adt_run_args(self, **kwargs):
136
62
        """Return list with the adt args for this provisioning backend."""
65
        """Return list with the adt args for this provisioning backend."""
137
63
        raise NotImplementedError()
66
        raise NotImplementedError()
138
64
67
139
=== modified file 'upgrade_testing/provisioning/backends/_qemu.py'
140
--- upgrade_testing/provisioning/backends/_qemu.py	2016-05-24 18:32:17 +0000
141
+++ upgrade_testing/provisioning/backends/_qemu.py	2017-07-14 19:31:31 +0000
142
@@ -18,16 +18,60 @@
143
18
18
144
19
import logging
19
import logging
145
20
import os
20
import os
146
21
import shlex
147
22
import shutil
148
23
import signal
149
24
import subprocess
150
25
import tempfile
151
26
import threading
152
27
153
28
from paramiko.ssh_exception import SSHException
154
21
29
155
22
from upgrade_testing.provisioning._util import run_command_with_logged_output
30
from upgrade_testing.provisioning._util import run_command_with_logged_output
157
23
from upgrade_testing.provisioning.backends._base import ProviderBackend
31
from upgrade_testing.provisioning.backends._ssh import SshBackend
158
24
32
159
25
CACHE_DIR = '/var/cache/auto-upgrade-testing'
33
CACHE_DIR = '/var/cache/auto-upgrade-testing'
160
34
OVERLAY_DIR = os.path.join(CACHE_DIR, 'overlay')
161
35
QEMU_LAUNCH_OPTS = (
162
36
    '{qemu} -m {ram} -smp {cpu} -pidfile {workdir}/qemu.pid -localtime '
163
37
    '-cpu core2duo -enable-kvm '
164
38
)
165
39
QEMU_SYSTEM_AMD64 = 'qemu-system-x86_64'
166
40
QEMU_SYSTEM_I386 = 'qemu-system-i386'
167
41
ARCH_AMD64 = 'amd64'
168
42
ARCH_I386 = 'i386'
169
43
QEMU_DISPLAY_OPTS = (
170
44
    '-display sdl '
171
45
)
172
46
QEMU_DISPLAY_VGA_OPTS = (
173
47
    '-vga qxl '
174
48
)
175
49
QEMU_SOUND_OPTS = (
176
50
    '-soundhw all '
177
51
)
178
52
QEMU_DISPLAY_HEADLESS = (
179
53
    '-display none '
180
54
)
181
55
QEMU_NET_OPTS = (
182
56
    '-net nic,model=virtio -net user'
183
57
)
184
58
QEMU_PORT_OPTS = (
185
59
    ',hostfwd=tcp::{port}-:22 '
186
60
)
187
61
QEMU_DISK_IMAGE_OPTS = (
188
62
    '-drive file={disk_img},if=virtio '
189
63
)
190
64
QEMU_DISK_IMAGE_OVERLAY_OPTS = (
191
65
    '-drive file={overlay_img},cache=unsafe,if=virtio,index=0 '
192
66
)
193
67
DEFAULT_RAM = '2G'
194
68
DEFAULT_CPU = '2'
195
69
HEADLESS = True
196
26
70
197
27
logger = logging.getLogger(__name__)
71
logger = logging.getLogger(__name__)
198
28
72
199
29
73
201
30
class QemuBackend(ProviderBackend):
74
class QemuBackend(SshBackend):
202
31
75
203
32
    # We can change the Backends to require just what they need. In this case
76
    # We can change the Backends to require just what they need. In this case
204
33
    # it would be distribution, release name (, arch)
77
    # it would be distribution, release name (, arch)
205
@@ -38,10 +82,14 @@
206
38
          details.
82
          details.
207
39
83
208
40
        """
84
        """
209
85
        super().__init__(release, arch, image_name, build_args)
210
41
        self.release = release
86
        self.release = release
211
42
        self.arch = arch
87
        self.arch = arch
212
43
        self.image_name = image_name
88
        self.image_name = image_name
213
44
        self.build_args = build_args
89
        self.build_args = build_args
214
90
        self.working_dir = None
215
91
        self.qemu_runner = None
216
92
        self.find_free_port()
217
45
93
218
46
    def available(self):
94
    def available(self):
219
47
        """Return true if a qemu exists that matches the provided args.
95
        """Return true if a qemu exists that matches the provided args.
220
@@ -73,9 +121,60 @@
221
73
        os.rename(initial_image_path, final_image_path)
121
        os.rename(initial_image_path, final_image_path)
222
74
        logger.info('Image created.')
122
        logger.info('Image created.')
223
75
123
225
76
    def get_adt_run_args(self, **kwargs):
124
    def close(self):
226
125
        if self.qemu_runner:
227
126
            try:
228
127
                self.shutdown()
229
128
            except PermissionError:
230
129
                print('Shutdown sudo command failed. '
231
130
                      'Check password: "{}".'.format(self.password))
232
131
                self.stop_qemu()
233
132
            except SSHException:
234
133
                self.stop_qemu()
235
134
            finally:
236
135
                self.qemu_runner.join(timeout=5)
237
136
            shutil.rmtree(self.working_dir)
238
137
            self.working_dir = None
239
138
            self.qemu_runner = None
240
139
            super().close()
241
140
242
141
    def reboot(self):
243
142
        self.close()
244
143
        self.connect()
245
144
246
145
    def stop_qemu(self):
247
146
        pid_file = os.path.join(self.working_dir, 'qemu.pid')
248
147
        with open(pid_file) as f:
249
148
            pid = int(f.read().strip())
250
149
        os.kill(pid, signal.SIGTERM)
251
150
252
151
    def get_adt_run_args(self, keep_overlay=False, **kwargs):
253
152
        if keep_overlay:
254
153
            self.qemu_runner = self.launch_qemu(
255
154
                self.image_name,
256
155
                kwargs.get('ram', DEFAULT_RAM),
257
156
                kwargs.get('cpu', DEFAULT_CPU),
258
157
                kwargs.get('headless', HEADLESS),
259
158
                port=self.port,
260
159
                overlay=os.path.join(OVERLAY_DIR,
261
160
                                     self.image_name))
262
161
            super().connect()
263
162
            return super().get_adt_run_args()
264
77
        return ['qemu', os.path.join(CACHE_DIR, self.image_name)]
163
        return ['qemu', os.path.join(CACHE_DIR, self.image_name)]
265
78
164
266
165
    def create_overlay_image(self, overlay_img):
267
166
        """Create an overlay image for specified base image."""
268
167
        overlay_dir = os.path.dirname(overlay_img)
269
168
        if os.path.isfile(overlay_img):
270
169
            os.remove(overlay_img)
271
170
        elif not os.path.isdir(overlay_dir):
272
171
            os.makedirs(overlay_dir)
273
172
        subprocess.check_call(
274
173
            ['qemu-img', 'create', '-f', 'qcow2', '-b',
275
174
             os.path.join(CACHE_DIR, self.image_name),
276
175
             overlay_img])
277
176
        subprocess.check_call(['sudo', 'chmod', '777', overlay_img])
278
177
279
79
    @property
178
    @property
280
80
    def name(self):
179
    def name(self):
281
81
        return 'qemu'
180
        return 'qemu'
282
@@ -85,3 +184,102 @@
283
85
            classname=self.__class__.__name__,
184
            classname=self.__class__.__name__,
284
86
            release=self.release
185
            release=self.release
285
87
        )
186
        )
286
187
287
188
    @staticmethod
288
189
    def get_architecture():
289
190
        """Return architecture string for system."""
290
191
        return subprocess.check_output(
291
192
            ['dpkg', '--print-architecture']).decode().strip()
292
193
293
194
    def get_qemu_path(self):
294
195
        """Return path of qemu-system executable for system."""
295
196
        if self.get_architecture() == ARCH_AMD64:
296
197
            target = QEMU_SYSTEM_AMD64
297
198
        else:
298
199
            target = QEMU_SYSTEM_I386
299
200
        return subprocess.check_output(['which', target]).decode().strip()
300
201
301
202
    def get_disk_args(self, overlay):
302
203
        """Return qemu-system disk args. If overlay is specified then an overlay
303
204
        image at that path will be created and specified in returned arguments.
304
205
        If no overlay is none then the base image will be returned in
305
206
        the arguments.
306
207
        :param overlay: Path of overlay image to use, otherwise None
307
208
        if not needed.
308
209
        :return: Disk image arguments as string.
309
210
        """
310
211
        if overlay:
311
212
            self.create_overlay_image(overlay)
312
213
            return QEMU_DISK_IMAGE_OVERLAY_OPTS.format(overlay_img=overlay)
313
214
        else:
314
215
            return QEMU_DISK_IMAGE_OPTS.format(disk_img=self.image_name)
315
216
316
217
    @staticmethod
317
218
    def get_display_args(headless):
318
219
        """Return qemu-system display arguments based on headless parameter.
319
220
        :param headless: Whether qemu-system should run in headless mode or not.
320
221
        :return: Display parameters for required display state.
321
222
        """
322
223
        if headless:
323
224
            return QEMU_DISPLAY_HEADLESS + QEMU_DISPLAY_VGA_OPTS
324
225
        else:
325
226
            return QEMU_DISPLAY_OPTS + QEMU_DISPLAY_VGA_OPTS + QEMU_SOUND_OPTS
326
227
327
228
    def launch_qemu(self, img, ram, cpu, headless, port, overlay):
328
229
        """Boot the qemu from a different thread to stop this thread from being
329
230
        blocked whilst the qemu is running.
330
231
        """
331
232
        self.working_dir = tempfile.mkdtemp()
332
233
        runner = threading.Thread(
333
234
            target=self._launch_qemu,
334
235
            args=(self.working_dir, img, ram, cpu, headless, port, overlay))
335
236
        runner.start()
336
237
        return runner
337
238
338
239
    def _launch_qemu(self, working_dir, disk_image_path, ram, cpu, headless,
339
240
                     port=None, overlay=None):
340
241
        """Launch qemu-system to install the iso file into the disk image.
341
242
        :param working_dir: Working directory to use.
342
243
        :param disk_image_path: Path of the disk image file used for
343
244
        installation.
344
245
        :param ram: Amount of ram allocated to qemu.
345
246
        :param cpu: Number of cpus allocated to qemu.
346
247
        :param headless: Whether to run installer in headless mode or not.
347
248
        :param port: Host port number to enable port forwarding to qemu port 22.
348
249
349
250
        """
350
251
        cmd = self.get_qemu_launch_command(
351
252
            working_dir, disk_image_path, ram, cpu, headless,
352
253
            port, overlay)
353
254
        print(' '.join(cmd))
354
255
        subprocess.check_call(cmd)
355
256
356
257
    def get_qemu_launch_command(self, work_dir, disk_img, ram, cpu,
357
258
                                headless, port=None, overlay=None):
358
259
        """Return command to launch qemu process using optional install parameters.
359
260
        :param work_dir: Working directory to use.
360
261
        :param disk_img: Path of the disk image file used for installation.
361
262
        :param ram: Amount of ram allocated to qemu.
362
263
        :param cpu: Number of cpus allocated to qemu.
363
264
        :param headless: Whether to run installer in headless mode or not.
364
265
        :return: Qemu launch command string.
365
266
        :param port: Host port number to enable port forwarding to qemu port 22.
366
267
        :param overlay: path to the overlay image to be created
367
268
        """
368
269
        # Create command base with resource parameters
369
270
        cmd = QEMU_LAUNCH_OPTS.format(
370
271
            qemu=self.get_qemu_path(), ram=ram, cpu=cpu, disk_img=disk_img,
371
272
            workdir=work_dir)
372
273
        # Get disk args including overlay image if specified
373
274
        cmd += self.get_disk_args(overlay)
374
275
        # Add display parameters
375
276
        cmd += self.get_display_args(headless)
376
277
        # Add network. This must preceed the port forwarding option.
377
278
        cmd += QEMU_NET_OPTS
378
279
        # Add port forwarding if specified
379
280
        if port:
380
281
            cmd += QEMU_PORT_OPTS.format(port=port)
381
282
        else:
382
283
            # Add space to separate options
383
284
            cmd += ' '
384
285
        return shlex.split(cmd)
385
88
286
386
=== added file 'upgrade_testing/provisioning/backends/_ssh.py'
387
--- upgrade_testing/provisioning/backends/_ssh.py	1970-01-01 00:00:00 +0000
388
+++ upgrade_testing/provisioning/backends/_ssh.py	2017-07-14 19:31:31 +0000
389
@@ -0,0 +1,260 @@
390
1
#
391
2
# Ubuntu Upgrade Testing
392
3
# Copyright (C) 2015 Canonical
393
4
#
394
5
# This program is free software: you can redistribute it and/or modify
395
6
# it under the terms of the GNU General Public License as published by
396
7
# the Free Software Foundation, either version 3 of the License, or
397
8
# (at your option) any later version.
398
9
#
399
10
# This program is distributed in the hope that it will be useful,
400
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
401
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
402
13
# GNU General Public License for more details.
403
14
#
404
15
# You should have received a copy of the GNU General Public License
405
16
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
406
17
#
407
18
408
19
import logging
409
20
import os
410
21
import socket
411
22
import subprocess
412
23
413
24
import time
414
25
415
26
import errno
416
27
import pexpect
417
28
from retrying import retry
418
29
419
30
from upgrade_testing.provisioning.backends._base import ProviderBackend
420
31
from upgrade_testing.provisioning.executors import SSHExecutor
421
32
422
33
CACHE_DIR = '/var/cache/auto-upgrade-testing'
423
34
424
35
logger = logging.getLogger(__name__)
425
36
TIMEOUT_CMD = 60
426
37
TIMEOUT_CONNECT = 120
427
38
TIMEOUT_WAIT_FOR_DEVICE = 120
428
39
429
40
430
41
class SshBackend(ProviderBackend):
431
42
432
43
    # We can change the Backends to require just what they need. In this case
433
44
    # it would be distribution, release name (, arch)
434
45
    def __init__(self, release, arch, image_name, build_args=[],
435
46
                 username=None, password=None, device_ip=None):
436
47
        """Provide backend capabilities as requested in the provision spec.
437
48
438
49
        :param provision_spec: ProvisionSpecification object containing backend
439
50
          details.
440
51
441
52
        """
442
53
        self.release = release
443
54
        self.arch = arch
444
55
        self.image_name = image_name
445
56
        self.build_args = build_args
446
57
        self.executor = SSHExecutor()
447
58
        self.username = username or 'ubuntu'
448
59
        self.password = password or 'ubuntu'
449
60
        self.connected = False
450
61
        self.key_file = None
451
62
        self.device_ip = device_ip or 'localhost'
452
63
        self.port = -1
453
64
454
65
    def available(self):
455
66
        """Return true if a qemu exists that matches the provided args.
456
67
457
68
        """
458
69
        image_name = self.image_name
459
70
        logger.info('Checking for {}'.format(image_name))
460
71
        return image_name in os.listdir(CACHE_DIR)
461
72
462
73
    def create(self, adt_base_path):
463
74
        raise NotImplementedError('Cannot create ssh backend directly')
464
75
465
76
    def get_adt_run_args(self, **kwargs):
466
77
        return ['ssh', '--port', str(self.port),
467
78
                '--login', self.username,
468
79
                '--password', self.password,
469
80
                '--identity', self.key_file,
470
81
                '--hostname',
471
82
                self.device_ip,
472
83
                '--reboot']
473
84
474
85
    @property
475
86
    def name(self):
476
87
        return 'ssh'
477
88
478
89
    def __repr__(self):
479
90
        return '{classname}(release={release})'.format(
480
91
            classname=self.__class__.__name__,
481
92
            release=self.release
482
93
        )
483
94
484
95
    port_from = 22220
485
96
    port_to = 23000
486
97
487
98
    def connect(self, timeout=TIMEOUT_CONNECT):
488
99
        if not self.connected:
489
100
            self.enable_ssh()
490
101
            self.executor.connect(
491
102
                self.username, self.password, self.port, self.device_ip,
492
103
                timeout)
493
104
            self.connected = True
494
105
495
106
    def close(self):
496
107
        if self.connected:
497
108
            self.executor.close()
498
109
            self.connected = False
499
110
500
111
    def reboot(self):
501
112
        self.executor.reboot()
502
113
503
114
    def shutdown(self):
504
115
        self.executor.shutdown()
505
116
506
117
    def put(self, src, dst):
507
118
        self.executor.put(src, dst)
508
119
509
120
    def run(self, command, timeout=TIMEOUT_CMD, log_stdout=True):
510
121
        return self.executor.run(command, timeout, log_stdout)
511
122
512
123
    def run_sudo(self, command, timeout=TIMEOUT_CMD, log_stdout=True):
513
124
        return self.executor.run_sudo(command, timeout, log_stdout)
514
125
515
126
    def find_free_port(self):
516
127
        for port in range(self.port_from, self.port_to):
517
128
            try:
518
129
                s = socket.create_connection(('127.0.0.1', port))
519
130
            except socket.error as e:
520
131
                if e.errno == errno.ECONNREFUSED:
521
132
                    # This means port is not currently used, so use this one
522
133
                    self.port = port
523
134
                    return
524
135
                else:
525
136
                    pass
526
137
            else:
527
138
                # port is already taken
528
139
                s.close()
529
140
        raise RuntimeError('Could not find free port for SSH connection.')
530
141
531
142
    def enable_ssh(self):
532
143
        """Enable ssh using public key."""
533
144
        self._wait_for_device()
534
145
        self._get_ssh_id_path()
535
146
        if not self._try_public_key_login():
536
147
            self._update_device_host_key()
537
148
            self._copy_ssh_id_to_device()
538
149
            self._verify_ssh_connect()
539
150
540
151
    def _wait_for_device(self, timeout=TIMEOUT_CONNECT):
541
152
        end = time.time() + timeout
542
153
        while time.time() < end:
543
154
            try:
544
155
                s = socket.create_connection(
545
156
                    (self.device_ip, str(self.port)), timeout)
546
157
            except ConnectionRefusedError:
547
158
                time.sleep(1)
548
159
            else:
549
160
                s.close()
550
161
                return
551
162
        raise TimeoutError(
552
163
            'Could not connect to {} '
553
164
            'port {}.'.format(self.device_ip, self.port))
554
165
555
166
    def _update_device_host_key(self):
556
167
        hosts_path = os.path.expanduser('~/.ssh/known_hosts')
557
168
        subprocess.call([
558
169
            'ssh-keygen', '-f', hosts_path, '-R',
559
170
            '[{}]:{}'.format(self.device_ip, self.port)])
560
171
561
172
    @retry(stop_max_attempt_number=20, wait_fixed=2000,
562
173
           retry_on_exception=lambda exception: isinstance(
563
174
               exception, RuntimeError))
564
175
    def _copy_ssh_id_to_device(self):
565
176
        pub_path = '{}.pub'.format(self.key_file)
566
177
        home_ssh = '/home/{u}/.ssh'.format(u=self.username)
567
178
        authorized_keys = os.path.join(home_ssh, 'authorized_keys')
568
179
        self._run(['mkdir', '-p', home_ssh])
569
180
        self._put(pub_path, authorized_keys)
570
181
        self._run(
571
182
            ['chown', '{u}:{u}'.format(u=self.username), '-R', home_ssh])
572
183
        self._run(['chmod', '700', home_ssh])
573
184
        self._run(['chmod', '600', authorized_keys])
574
185
575
186
    def _get_ssh_id_path(self):
576
187
        match = False
577
188
        for id in ['~/.ssh/id_rsa', '~/.ssh/id_autopkgtest']:
578
189
            path = os.path.expanduser(id)
579
190
            if os.path.exists(path):
580
191
                match = True
581
192
                break
582
193
        if not match:
583
194
            subprocess.check_call([
584
195
                'ssh-keygen', '-q', '-t', 'rsa', '-f', path, '-N', ''])
585
196
        self.key_file = path
586
197
587
198
    def _try_public_key_login(self):
588
199
        """Try and log in using public key. If this succeeds then there is no
589
200
        need to do any further ssh setup.
590
201
        :return: True if login was successful, False otherwise
591
202
        """
592
203
        cmd = ' '.join(['ssh', '-p', str(self.port),
593
204
                        '-o', 'UserKnownHostsFile=/dev/null',
594
205
                        '-o', 'StrictHostKeyChecking=no',
595
206
                        '-i', self.key_file, '-l', self.username,
596
207
                        self.device_ip])
597
208
        child = pexpect.spawn(cmd)
598
209
        try:
599
210
            index = child.expect(
600
211
                ['\$', 'password', 'denied'], timeout=TIMEOUT_CONNECT)
601
212
        except (pexpect.exceptions.TIMEOUT, pexpect.exceptions.EOF):
602
213
            index = -1
603
214
        finally:
604
215
            child.close()
605
216
        return index == 0
606
217
607
218
    def _verify_ssh_connect(self):
608
219
        """Verify that an ssh connection can be established without using
609
220
        password.
610
221
        """
611
222
        for count in range(20):
612
223
            if self._try_public_key_login():
613
224
                return
614
225
            else:
615
226
                time.sleep(1)
616
227
        raise RuntimeError('Could not create ssh connection')
617
228
618
229
    def _run(self, commands, timeout=TIMEOUT_CMD):
619
230
        """Run a command setting up an ssh connection using password."""
620
231
        ssh_cmd = [
621
232
            'ssh', '-o', 'StrictHostKeyChecking=no', '-p', str(self.port),
622
233
            '{}@{}'.format(self.username, self.device_ip)]
623
234
        self._run_with_password(ssh_cmd + commands, self.password, timeout)
624
235
625
236
    def _put(self, src, dst, timeout=TIMEOUT_CMD):
626
237
        """Put file onto device setting up an ssh connection using password."""
627
238
        scp_cmd = [
628
239
            'scp', '-o', 'StrictHostKeyChecking=no', '-P', str(self.port),
629
240
            src, '{}@{}:{}'.format(self.username, self.device_ip, dst)]
630
241
        self._run_with_password(scp_cmd, self.password, timeout)
631
242
632
243
    def _run_with_password(self, commands, password, timeout=TIMEOUT_CMD):
633
244
        """Run command expecting a password prompt to be displayed."""
634
245
        command = ' '.join(commands)
635
246
        child = pexpect.spawn(command)
636
247
        try:
637
248
            child.expect('password', timeout=timeout)
638
249
        except pexpect.exceptions.EOF:
639
250
            # No password prompt is displayed, so just continue
640
251
            pass
641
252
        else:
642
253
            child.sendline(password)
643
254
            if child.expect([pexpect.EOF, 'denied'], timeout=timeout):
644
255
                raise PermissionError(
645
256
                    'Check password is correct: "{}"'.format(password))
646
257
        finally:
647
258
            child.close()
648
259
        if child.exitstatus:
649
260
            raise RuntimeError('Error running {}'.format(command))
650
0
261
651
=== added file 'upgrade_testing/provisioning/executors.py'
652
--- upgrade_testing/provisioning/executors.py	1970-01-01 00:00:00 +0000
653
+++ upgrade_testing/provisioning/executors.py	2017-07-14 19:31:31 +0000
654
@@ -0,0 +1,206 @@
655
1
#
656
2
# Ubuntu Upgrade Testing
657
3
# Copyright (C) 2017 Canonical
658
4
#
659
5
# This program is free software: you can redistribute it and/or modify
660
6
# it under the terms of the GNU General Public License as published by
661
7
# the Free Software Foundation, either version 3 of the License, or
662
8
# (at your option) any later version.
663
9
#
664
10
# This program is distributed in the hope that it will be useful,
665
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
666
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
667
13
# GNU General Public License for more details.
668
14
#
669
15
# You should have received a copy of the GNU General Public License
670
16
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
671
17
#
672
18
673
19
674
20
import os
675
21
import sys
676
22
import time
677
23
from abc import ABCMeta
678
24
from abc import abstractmethod
679
25
680
26
import paramiko
681
27
682
28
TIMEOUT_CMD = 60
683
29
TIMEOUT_CONNECT = 120
684
30
TIMEOUT_WAIT_FOR_DEVICE = 120
685
31
686
32
687
33
class Result:
688
34
    """Result of command with status and output properties."""
689
35
690
36
    def __init__(self):
691
37
        self.status = None
692
38
        self.output = ''
693
39
694
40
695
41
class SSHClient:
696
42
    """This class manages the paramiko ssh client"""
697
43
698
44
    def __init__(self):
699
45
        """ The ssh can be initialized either through a password or with a
700
46
        private key file and the passphrase
701
47
        :param hostname: The hostname to connect
702
48
        :param user: The remote user in the host
703
49
        :param keyfile: The private key used to connect to the remote host
704
50
        :param password: The password used to connect to the remote host
705
51
        :param timeout: An optional timeout (in seconds) for the TCP connect
706
52
        """
707
53
        self.client = paramiko.SSHClient()
708
54
        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
709
55
710
56
    def connect(self, hostname, user, password, port, timeout=60):
711
57
        """Connect to remote host."""
712
58
        timeout = float(timeout)
713
59
        self.client.connect(
714
60
            hostname, username=user, password=password, port=port,
715
61
            timeout=timeout, banner_timeout=timeout)
716
62
717
63
    def close(self):
718
64
        """Close the connection"""
719
65
        self.client.close()
720
66
721
67
    def run(self, cmd, timeout=TIMEOUT_CMD, log_stdout=True):
722
68
        """Run a command in the remote host.
723
69
        :param cmd: Command to run.
724
70
        :param timeout: Period to wait before raising TimeoutError.
725
71
        :param log_stdout: Whether to log output to stdout.
726
72
        :return: Result object containing command output and status code.
727
73
        """
728
74
        channel = self.client.get_transport().open_session()
729
75
        channel.set_combine_stderr(True)
730
76
        end = time.time() + timeout
731
77
        result = Result()
732
78
        channel.exec_command(cmd)
733
79
        while (not channel.exit_status_ready() or (
734
80
               channel.exit_status_ready() and channel.recv_ready())):
735
81
            if channel.recv_ready():
736
82
                self._process_output(
737
83
                    result, log_stdout, channel.recv(1024).decode())
738
84
            elif time.time() > end:
739
85
                print('Timeout waiting {} seconds for command to '
740
86
                      'complete: {}'.format(timeout, cmd))
741
87
                raise TimeoutError
742
88
            time.sleep(0.2)
743
89
        # Add a new line after command has completed to ensure output
744
90
        # is separated.
745
91
        self._process_output(result, log_stdout, '\n')
746
92
        result.status = channel.recv_exit_status()
747
93
        return result
748
94
749
95
    def _process_output(self, result, log_stdout, content):
750
96
        """Save output to result and print to stdout if required."""
751
97
        result.output += content
752
98
        if log_stdout:
753
99
            sys.stdout.write(content)
754
100
            sys.stdout.flush()
755
101
756
102
    def put(self, local_path, remote_path):
757
103
        """Copy a file through sftp from the local_path to the remote_path"""
758
104
        if not os.path.isfile(local_path):
759
105
            raise RuntimeError('File to copy does not exist')
760
106
761
107
        with self.client.open_sftp() as sftp:
762
108
            sftp.put(local_path, remote_path)
763
109
764
110
    def get(self, remote_path, local_path):
765
111
        """Copy a file through sftp from the remote_path to the local_path"""
766
112
        with self.client.open_sftp() as sftp:
767
113
            sftp.get(remote_path, local_path)
768
114
769
115
        if not os.path.isfile(local_path):
770
116
            raise RuntimeError('File couldn\'t be copied')
771
117
772
118
773
119
class Executor:
774
120
    __metaclass__ = ABCMeta
775
121
    """Base class for all target executors."""
776
122
777
123
    @abstractmethod
778
124
    def connect(self, username, password, port, host=None, timeout=None):
779
125
        pass
780
126
781
127
    @abstractmethod
782
128
    def close(self):
783
129
        pass
784
130
785
131
    @abstractmethod
786
132
    def run(self, cmd, timeout=None, log_stdout=True):
787
133
        pass
788
134
789
135
    @abstractmethod
790
136
    def run_sudo(self, cmd, timeout=None, log_stdout=True):
791
137
        pass
792
138
793
139
    def reboot(self):
794
140
        result = self.run_sudo('shutdown -r now')
795
141
        if result.status > 0:
796
142
            raise PermissionError('Reboot failed, check password.')
797
143
798
144
    def shutdown(self):
799
145
        result = self.run_sudo('shutdown now')
800
146
        if result.status > 0:
801
147
            raise PermissionError('Shutdown failed, check password.')
802
148
803
149
    @abstractmethod
804
150
    def wait_for_device(self, timeout=None):
805
151
        pass
806
152
807
153
    @abstractmethod
808
154
    def put(self, localpath, remotepath):
809
155
        pass
810
156
811
157
    @abstractmethod
812
158
    def get(self, remotepath, localpath):
813
159
        pass
814
160
815
161
    def _get_sudo_command(self, cmd):
816
162
        command = 'sudo {}'.format(cmd)
817
163
        if self.password:
818
164
            command = 'echo {} | sudo -S {}'.format(self.password, cmd)
819
165
        return command
820
166
821
167
822
168
class SSHExecutor(Executor):
823
169
824
170
    def __init__(self):
825
171
        self.ssh_client = SSHClient()
826
172
827
173
    def connect(self, username, password, port, host='localhost',
828
174
                timeout=TIMEOUT_CONNECT):
829
175
        self.password = password
830
176
        count = max(1, timeout)
831
177
        for attempt in range(count):
832
178
            try:
833
179
                self.ssh_client.connect(
834
180
                    host, username, password, port, timeout)
835
181
            except TypeError:
836
182
                # This can happen when target not yet running so just try again
837
183
                time.sleep(1)
838
184
            except paramiko.ssh_exception.AuthenticationException:
839
185
                raise
840
186
            else:
841
187
                return
842
188
        raise RuntimeError('Could not connect to target.')
843
189
844
190
    def close(self):
845
191
        self.ssh_client.close()
846
192
847
193
    def _run(self, cmd, timeout=TIMEOUT_CMD, log_stdout=True):
848
194
        return self.ssh_client.run(cmd, timeout, log_stdout)
849
195
850
196
    def run(self, cmd, timeout=TIMEOUT_CMD, log_stdout=True):
851
197
        return self._run(cmd, timeout, log_stdout)
852
198
853
199
    def run_sudo(self, cmd, timeout=TIMEOUT_CMD, log_stdout=True):
854
200
        return self._run(self._get_sudo_command(cmd), timeout, log_stdout)
855
201
856
202
    def put(self, localpath, remotepath):
857
203
        self.ssh_client.put(localpath, remotepath)
858
204
859
205
    def get(self, remotepath, localpath):
860
206
        self.ssh_client.get(remotepath, localpath)
Status:	Merged
Approved by:	Jean-Baptiste Lallement on 2017-07-17
Approved revision:	96
Merged at revision:	89
Proposed branch:	lp:~heber013/auto-upgrade-testing/adding-keep-overlay-option
Merge into:	lp:auto-upgrade-testing
Diff against target:	860 lines (+714/-8) 8 files modified .flake8 (+5/-0) .project-flake8.sh (+19/-1) debian/control (+6/-0) upgrade_testing/command_line.py (+14/-4) upgrade_testing/provisioning/_provisionconfig.py (+3/-0) upgrade_testing/provisioning/backends/_qemu.py (+201/-3) upgrade_testing/provisioning/backends/_ssh.py (+260/-0) upgrade_testing/provisioning/executors.py (+206/-0)
To merge this branch:	bzr merge lp:~heber013/auto-upgrade-testing/adding-keep-overlay-option
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
Jean-Baptiste Lallement		2017-07-14	Approve on 2017-07-17
platform-qa-bot	continuous-integration		Approve on 2017-07-14
Review via email: mp+327450@code.launchpad.net