1
diff --git a/curtin/commands/extract.py b/curtin/commands/extract.py
2
index ec7a791..8567659 100644
3
--- a/curtin/commands/extract.py
4
+++ b/curtin/commands/extract.py
5
@@ -1,6 +1,7 @@
6
1
# This file is part of curtin. See LICENSE file for copyright and license info.
1
# This file is part of curtin. See LICENSE file for copyright and license info.
7
2
2
8
3
import os
3
import os
9
4
import shutil
10
4
import sys
5
import sys
11
5
import tempfile
6
import tempfile
12
6
7
13
@@ -64,7 +65,7 @@ def extract_root_fsimage_url(url, target):
14
64
    wfp = tempfile.NamedTemporaryFile(suffix=".img", delete=False)
65
    wfp = tempfile.NamedTemporaryFile(suffix=".img", delete=False)
15
65
    wfp.close()
66
    wfp.close()
16
66
    try:
67
    try:
18
67
        url_helper.download(url, wfp.name)
68
        url_helper.download(url, wfp.name, retries=3)
19
68
        return _extract_root_fsimage(wfp.name, target)
69
        return _extract_root_fsimage(wfp.name, target)
20
69
    finally:
70
    finally:
21
70
        os.unlink(wfp.name)
71
        os.unlink(wfp.name)
22
@@ -85,6 +86,121 @@ def _extract_root_fsimage(path, target):
23
85
        os.rmdir(mp)
86
        os.rmdir(mp)
24
86
87
25
87
88
26
89
def extract_root_layered_fsimage_url(uri, target):
27
90
    ''' Build images list to consider from a layered structure
28
91
29
92
    uri: URI of the layer file
30
93
    target: Target file system to provision
31
94
32
95
    return: None
33
96
    '''
34
97
    path = _path_from_file_url(uri)
35
98
36
99
    image_stack = _get_image_stack(path)
37
100
    LOG.debug("Considering fsimages: '%s'", ",".join(image_stack))
38
101
39
102
    tmp_dir = None
40
103
    try:
41
104
        # Download every remote images if remote url
42
105
        if url_helper.urlparse(path).scheme != "":
43
106
            tmp_dir = tempfile.mkdtemp()
44
107
            image_stack = _download_layered_images(image_stack, tmp_dir)
45
108
46
109
        # Check that all images exists on disk and are not empty
47
110
        for img in image_stack:
48
111
            if not os.path.isfile(img) or os.path.getsize(img) <= 0:
49
112
                raise ValueError("Failed to use fsimage: '%s' doesn't exist " +
50
113
                                 "or is invalid", img)
51
114
52
115
        return _extract_root_layered_fsimage(image_stack, target)
53
116
    finally:
54
117
        if tmp_dir and os.path.exists(tmp_dir):
55
118
            shutil.rmtree(tmp_dir)
56
119
57
120
58
121
def _download_layered_images(image_stack, tmp_dir):
59
122
    local_image_stack = []
60
123
    try:
61
124
        for img_url in image_stack:
62
125
            dest_path = os.path.join(tmp_dir,
63
126
                                     os.path.basename(img_url))
64
127
            url_helper.download(img_url, dest_path, retries=3)
65
128
            local_image_stack.append(dest_path)
66
129
    except url_helper.UrlError as e:
67
130
        LOG.error("Failed to download '%s'" % img_url)
68
131
        raise e
69
132
    return local_image_stack
70
133
71
134
72
135
def _extract_root_layered_fsimage(image_stack, target):
73
136
    mp_base = tempfile.mkdtemp()
74
137
    mps = []
75
138
    try:
76
139
        # Create a mount point for each image file and mount the image
77
140
        try:
78
141
            for img in image_stack:
79
142
                mp = os.path.join(mp_base, os.path.basename(img) + ".dir")
80
143
                os.mkdir(mp)
81
144
                util.subp(['mount', '-o', 'loop,ro', img, mp], capture=True)
82
145
                mps.insert(0, mp)
83
146
        except util.ProcessExecutionError as e:
84
147
            LOG.error("Failed to mount '%s' for extraction: %s", img, e)
85
148
            raise e
86
149
87
150
        # Prepare
88
151
        if len(mps) == 1:
89
152
            root_dir = mps[0]
90
153
        else:
91
154
            # Multiple image files, merge them with an overlay and do the copy
92
155
            root_dir = os.path.join(mp_base, "root.dir")
93
156
            os.mkdir(root_dir)
94
157
            try:
95
158
                util.subp(['mount', '-t', 'overlay', 'overlay', '-o',
96
159
                           'lowerdir=' + ':'.join(mps), root_dir],
97
160
                          capture=True)
98
161
                mps.append(root_dir)
99
162
            except util.ProcessExecutionError as e:
100
163
                LOG.error("overlay mount to %s failed: %s", root_dir, e)
101
164
                raise e
102
165
103
166
        copy_to_target(root_dir, target)
104
167
    finally:
105
168
        umount_err_mps = []
106
169
        for mp in reversed(mps):
107
170
            try:
108
171
                util.subp(['umount', mp], capture=True)
109
172
            except util.ProcessExecutionError as e:
110
173
                LOG.error("can't unmount %s: %e", mp, e)
111
174
                umount_err_mps.append(mp)
112
175
        if umount_err_mps:
113
176
            raise util.ProcessExecutionError(
114
177
                "Failed to umount: %s", ", ".join(umount_err_mps))
115
178
        shutil.rmtree(mp_base)
116
179
117
180
118
181
def _get_image_stack(uri):
119
182
    '''Find a list of dependent images for given layered fsimage path
120
183
121
184
    uri: URI of the layer file
122
185
    return: tuple of path to dependent images
123
186
    '''
124
187
125
188
    image_stack = []
126
189
    root_dir = os.path.dirname(uri)
127
190
    img_name = os.path.basename(uri)
128
191
    _, img_ext = os.path.splitext(img_name)
129
192
130
193
    img_parts = img_name.split('.')
131
194
    # Last item is the extension
132
195
    for i in img_parts[:-1]:
133
196
        image_stack.append(
134
197
            os.path.join(
135
198
                root_dir,
136
199
                '.'.join(img_parts[0:img_parts.index(i)+1]) + img_ext))
137
200
138
201
    return image_stack
139
202
140
203
141
88
def copy_to_target(source, target):
204
def copy_to_target(source, target):
142
89
    if source.startswith("cp://"):
205
    if source.startswith("cp://"):
143
90
        source = source[5:]
206
        source = source[5:]
144
@@ -133,6 +249,8 @@ def extract(args):
145
133
                copy_to_target(source['uri'], target)
249
                copy_to_target(source['uri'], target)
146
134
            elif source['type'] == "fsimage":
250
            elif source['type'] == "fsimage":
147
135
                extract_root_fsimage_url(source['uri'], target=target)
251
                extract_root_fsimage_url(source['uri'], target=target)
148
252
            elif source['type'] == "fsimage-layered":
149
253
                extract_root_layered_fsimage_url(source['uri'], target=target)
150
136
            else:
254
            else:
151
137
                extract_root_tgz_url(source['uri'], target=target)
255
                extract_root_tgz_url(source['uri'], target=target)
152
138
256
153
diff --git a/curtin/commands/install.py b/curtin/commands/install.py
154
index 244683c..ad17508 100644
155
--- a/curtin/commands/install.py
156
+++ b/curtin/commands/install.py
157
@@ -408,10 +408,6 @@ def cmd_install(args):
158
408
    if not len(cfg.get('sources', [])):
408
    if not len(cfg.get('sources', [])):
159
409
        raise util.BadUsage("no sources provided to install")
409
        raise util.BadUsage("no sources provided to install")
160
410
410
161
411
    for i in cfg['sources']:
162
412
        # we default to tgz for old style sources config
163
413
        cfg['sources'][i] = util.sanitize_source(cfg['sources'][i])
164
414
165
415
    migrate_proxy_settings(cfg)
411
    migrate_proxy_settings(cfg)
166
416
    for k in ('http_proxy', 'https_proxy', 'no_proxy'):
412
    for k in ('http_proxy', 'https_proxy', 'no_proxy'):
167
417
        if k in cfg['proxy']:
413
        if k in cfg['proxy']:
168
diff --git a/curtin/url_helper.py b/curtin/url_helper.py
169
index 767510d..ee4640b 100644
170
--- a/curtin/url_helper.py
171
+++ b/curtin/url_helper.py
172
@@ -82,39 +82,51 @@ class UrlReader(object):
173
82
        self.close()
82
        self.close()
174
83
83
175
84
84
177
85
def download(url, path, reporthook=None, data=None):
85
def download(url, path, reporthook=None, data=None, retries=0, retry_delay=3):
178
86
    """Download url to path.
86
    """Download url to path.
179
87
87
180
88
    reporthook is compatible with py3 urllib.request.urlretrieve.
88
    reporthook is compatible with py3 urllib.request.urlretrieve.
181
89
    urlretrieve does not exist in py2."""
89
    urlretrieve does not exist in py2."""
182
90
90
183
91
    buflen = 8192
91
    buflen = 8192
185
92
    wfp = open(path, "wb")
92
    attempts = 0
186
93
93
201
94
    try:
94
    while True:
202
95
        buf = None
95
        wfp = open(path, "wb")
203
96
        blocknum = 0
96
        try:
204
97
        fsize = 0
97
            buf = None
205
98
        start = time.time()
98
            blocknum = 0
206
99
        with UrlReader(url) as rfp:
99
            fsize = 0
207
100
            if reporthook:
100
            start = time.time()
208
101
                reporthook(blocknum, buflen, rfp.size)
101
            with UrlReader(url) as rfp:
195
102
196
103
            while True:
197
104
                buf = rfp.read(buflen)
198
105
                if not buf:
199
106
                    break
200
107
                blocknum += 1
209
108
                if reporthook:
102
                if reporthook:
210
109
                    reporthook(blocknum, buflen, rfp.size)
103
                    reporthook(blocknum, buflen, rfp.size)
219
110
                wfp.write(buf)
104
220
111
                fsize += len(buf)
105
                while True:
221
112
        timedelta = time.time() - start
106
                    buf = rfp.read(buflen)
222
113
        LOG.debug("Downloaded %d bytes from %s to %s in %.2fs (%.2fMbps)",
107
                    if not buf:
223
114
                  fsize, url, path, timedelta, fsize / timedelta / 1024 / 1024)
108
                        break
224
115
        return path, rfp.info
109
                    blocknum += 1
225
116
    finally:
110
                    if reporthook:
226
117
        wfp.close()
111
                        reporthook(blocknum, buflen, rfp.size)
227
112
                    wfp.write(buf)
228
113
                    fsize += len(buf)
229
114
            timedelta = time.time() - start
230
115
            LOG.debug("Downloaded %d bytes from %s to %s in %.2fs (%.2fMbps)",
231
116
                      fsize, url, path, timedelta,
232
117
                      fsize / timedelta / 1024 / 1024)
233
118
            return path, rfp.info
234
119
        except UrlError as e:
235
120
            # retry on internal server errors up to "retries #"
236
121
            if (e.code < 500 or attempts >= retries):
237
122
                raise e
238
123
            LOG.debug("Current download failed with error: %s. Retrying in"
239
124
                      " %d seconds.",
240
125
                      e.code, retry_delay)
241
126
            attempts += 1
242
127
            time.sleep(retry_delay)
243
128
        finally:
244
129
            wfp.close()
245
118
130
246
119
131
247
120
def get_maas_version(endpoint):
132
def get_maas_version(endpoint):
248
diff --git a/curtin/util.py b/curtin/util.py
249
index 35e004f..34d09a8 100644
250
--- a/curtin/util.py
251
+++ b/curtin/util.py
252
@@ -45,6 +45,10 @@ except NameError:
253
45
from . import paths
45
from . import paths
254
46
from .log import LOG, log_call
46
from .log import LOG, log_call
255
47
47
256
48
binary_type = bytes
257
49
if sys.version_info[0] < 3:
258
50
    binary_type = str
259
51
260
48
_INSTALLED_HELPERS_PATH = 'usr/lib/curtin/helpers'
52
_INSTALLED_HELPERS_PATH = 'usr/lib/curtin/helpers'
261
49
_INSTALLED_MAIN = 'usr/bin/curtin'
53
_INSTALLED_MAIN = 'usr/bin/curtin'
262
50
54
263
@@ -891,7 +895,7 @@ def sanitize_source(source):
264
891
        # already sanitized?
895
        # already sanitized?
265
892
        return source
896
        return source
266
893
    supported = ['tgz', 'dd-tgz', 'dd-tbz', 'dd-txz', 'dd-tar', 'dd-bz2',
897
    supported = ['tgz', 'dd-tgz', 'dd-tbz', 'dd-txz', 'dd-tar', 'dd-bz2',
268
894
                 'dd-gz', 'dd-xz', 'dd-raw', 'fsimage']
898
                 'dd-gz', 'dd-xz', 'dd-raw', 'fsimage', 'fsimage-layered']
269
895
    deftype = 'tgz'
899
    deftype = 'tgz'
270
896
    for i in supported:
900
    for i in supported:
271
897
        prefix = i + ":"
901
        prefix = i + ":"
272
diff --git a/doc/topics/config.rst b/doc/topics/config.rst
273
index 218bc17..bad8fc2 100644
274
--- a/doc/topics/config.rst
275
+++ b/doc/topics/config.rst
276
@@ -506,9 +506,83 @@ configures the method used to copy the image to the target system.
277
506
- **cp://**: Use ``rsync`` command to copy source directory to target.
506
- **cp://**: Use ``rsync`` command to copy source directory to target.
278
507
- **file://**: Use ``tar`` command to extract source to target.
507
- **file://**: Use ``tar`` command to extract source to target.
279
508
- **http[s]://**: Use ``wget | tar`` commands to extract source to target.
508
- **http[s]://**: Use ``wget | tar`` commands to extract source to target.
282
509
- **fsimage://**: mount filesystem image and copy contents to target.
509
- **fsimage://** mount filesystem image and copy contents to target.
283
510
  Local file or url are supported.  Filesystem can be any filesystem type
510
  Local file or url are supported. Filesystem can be any filesystem type
284
511
  mountable by the running kernel.
511
  mountable by the running kernel.
285
512
- **fsimage-layered://** mount layered filesystem image and copy contents to target.
286
513
  A ``fsimage-layered`` install source is a string representing one or more mountable
287
514
  images from a single local or remote directory.  The string is dot-separated where
288
515
  each value between the dots represents a particular image and the location of the
289
516
  name within the string encodes the order in which it is to be mounted.  The resulting
290
517
  list of images are downloaded (if needed) then mounted and overlayed into a single
291
518
  directory which is used as the source for installation.
292
519
293
520
**Image Name Pattern**
294
521
295
522
 [[<parent_layer>.]...]<layer name>.<file extension pattern>
296
523
297
524
Example::
298
525
299
526
 10-base.img
300
527
 minimal.img
301
528
 minimal.standard.live.squashfs
302
529
 http://example.io/standard.squashfs
303
530
304
531
**Layer Dependencies**
305
532
306
533
Layers are parts of the name seperated by dots. Any layer in the name will
307
534
be included as a dependency. The file extension pattern is used to find
308
535
related layers.
309
536
310
537
Examples:
311
538
312
539
 Base use case::
313
540
314
541
  /images
315
542
  ├── main.squashfs
316
543
  ├── main.upper.squashfs
317
544
  └── main.upper.debug.squashfs
318
545
319
546
 source='fsimage-layered://images/main.squashfs' -> images='/images/main.squashfs'
320
547
 source='fsimage-layered://images/main.upper.squashfs' -> images='/images/main.upper.squashfs, /images/main.squashfs'
321
548
 source='fsimage-layered://images/main.upper.debug.squashfs' -> images='/images/main.upper.debug.squashfs, /images/main.upper.squashfs, /images/main.squashfs'
322
549
323
550
 Multiple extensions::
324
551
325
552
  /images
326
553
  ├── main.squashfs
327
554
  ├── main.img
328
555
  ├── main.upper.squashfs
329
556
  ├── main.upper.img
330
557
  └── main.upper.debug.squashfs
331
558
332
559
 source='fsimage-layered://images/main.upper.squashfs' -> images='/images/main.upper.squashfs, /images/main.squashfs'
333
560
 source='fsimage-layered://images/main.upper.img' -> images='/images/main.upper.img, /images/main.img'
334
561
335
562
 Missing intermediary layer::
336
563
337
564
  /images
338
565
  ├── main.squashfs
339
566
  └── main.upper.debug.squashfs
340
567
341
568
If there is a missing image in the path to a leaf, an error will be raised
342
569
343
570
 source='fsimage-layered://images/main.squashfs' -> images='/images/main.squashfs'
344
571
 source='fsimage-layered://images/main.upper.debug.squashfs' -> Raised Error'
345
572
346
573
 Remote Layers::
347
574
348
575
  http://example.io/base.extended.debug.squashfs
349
576
350
577
351
578
The URI passed to ``fsimage-layered`` may be on a remote system.  Curtin
352
579
will parse the URI and then download each layer from the remote system.
353
580
This results in Curtin downloading the following URLs::
354
581
355
582
- http://example.io/base.squashfs
356
583
- http://example.io/base.extended.squashfs
357
584
- http://example.io/base.extended.debug.squashfs
358
585
359
512
586
360
513
**Example Cloud-image**::
587
**Example Cloud-image**::
361
514
588
362
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
363
index 1268880..0963483 100644
364
--- a/tests/unittests/helpers.py
365
+++ b/tests/unittests/helpers.py
366
@@ -11,6 +11,8 @@ import string
367
11
import tempfile
11
import tempfile
368
12
from unittest import TestCase
12
from unittest import TestCase
369
13
13
370
14
from curtin import util
371
15
372
14
16
373
15
def builtin_module_name():
17
def builtin_module_name():
374
16
    options = ('builtins', '__builtin__')
18
    options = ('builtins', '__builtin__')
375
@@ -87,4 +89,22 @@ def dir2dict(startdir, prefix=None):
376
87
                flist[key] = fp.read()
89
                flist[key] = fp.read()
377
88
    return flist
90
    return flist
378
89
91
379
92
380
93
def populate_dir(path, files):
381
94
    if not os.path.exists(path):
382
95
        os.makedirs(path)
383
96
    ret = []
384
97
    for (name, content) in files.items():
385
98
        p = os.path.sep.join([path, name])
386
99
        util.ensure_dir(os.path.dirname(p))
387
100
        with open(p, "wb") as fp:
388
101
            if isinstance(content, util.binary_type):
389
102
                fp.write(content)
390
103
            else:
391
104
                fp.write(content.encode('utf-8'))
392
105
            fp.close()
393
106
        ret.append(p)
394
107
395
108
    return ret
396
109
397
90
# vi: ts=4 expandtab syntax=python
110
# vi: ts=4 expandtab syntax=python
398
diff --git a/tests/unittests/test_commands_extract.py b/tests/unittests/test_commands_extract.py
399
index cc117bb..c318699 100644
400
--- a/tests/unittests/test_commands_extract.py
401
+++ b/tests/unittests/test_commands_extract.py
402
@@ -4,12 +4,15 @@ import os
403
4
from .helpers import CiTestCase
4
from .helpers import CiTestCase
404
5
5
405
6
from curtin import util
6
from curtin import util
407
7
from curtin.commands.extract import extract_root_fsimage_url
7
from curtin.commands.extract import (extract_root_fsimage_url,
408
8
                                     extract_root_layered_fsimage_url,
409
9
                                     _get_image_stack)
410
10
from curtin.url_helper import UrlError
411
8
11
412
9
12
413
10
class TestExtractRootFsImageUrl(CiTestCase):
13
class TestExtractRootFsImageUrl(CiTestCase):
414
11
    """Test extract_root_fsimage_url."""
14
    """Test extract_root_fsimage_url."""
416
12
    def _fake_download(self, url, path):
15
    def _fake_download(self, url, path, retries=0):
417
13
        self.downloads.append(os.path.abspath(path))
16
        self.downloads.append(os.path.abspath(path))
418
14
        with open(path, "w") as fp:
17
        with open(path, "w") as fp:
419
15
            fp.write("fake content from " + url + "\n")
18
            fp.write("fake content from " + url + "\n")
420
@@ -43,7 +46,7 @@ class TestExtractRootFsImageUrl(CiTestCase):
421
43
        target = self.tmp_path("target_d", tmpd)
46
        target = self.tmp_path("target_d", tmpd)
422
44
        fpath = self.tmp_path("my.img", tmpd)
47
        fpath = self.tmp_path("my.img", tmpd)
423
45
        util.write_file(fpath, fpath + " data\n")
48
        util.write_file(fpath, fpath + " data\n")
425
46
        extract_root_fsimage_url("file://" + fpath, target)
49
        extract_root_fsimage_url("file:///" + fpath, target)
426
47
        self.assertEqual(1, self.m__extract_root_fsimage.call_count)
50
        self.assertEqual(1, self.m__extract_root_fsimage.call_count)
427
48
        self.assertEqual(0, self.m_download.call_count)
51
        self.assertEqual(0, self.m_download.call_count)
428
49
52
429
@@ -70,4 +73,225 @@ class TestExtractRootFsImageUrl(CiTestCase):
430
70
        self.assertEqual(0, self.m_download.call_count)
73
        self.assertEqual(0, self.m_download.call_count)
431
71
74
432
72
75
433
76
class TestExtractRootLayeredFsImageUrl(CiTestCase):
434
77
    """Test extract_root_layared_fsimage_url."""
435
78
    def _fake_download(self, url, path, retries=0):
436
79
        self.downloads.append(os.path.abspath(path))
437
80
        with open(path, "w") as fp:
438
81
            fp.write("fake content from " + url + "\n")
439
82
440
83
    def setUp(self):
441
84
        super(TestExtractRootLayeredFsImageUrl, self).setUp()
442
85
        self.downloads = []
443
86
        self.add_patch("curtin.commands.extract.url_helper.download",
444
87
                       "m_download", side_effect=self._fake_download)
445
88
        self.add_patch("curtin.commands.extract._extract_root_layered_fsimage",
446
89
                       "m__extract_root_layered_fsimage")
447
90
448
91
    def test_relative_local_file_single(self):
449
92
        """extract_root_layered_fsimage_url supports relative file:// uris."""
450
93
        tmpd = self.tmp_dir()
451
94
        target = self.tmp_path("target_d", tmpd)
452
95
        startdir = os.getcwd()
453
96
        fname = "my.img"
454
97
        try:
455
98
            os.chdir(tmpd)
456
99
            util.write_file(fname, fname + " data\n")
457
100
            extract_root_layered_fsimage_url("file://" + fname, target)
458
101
        finally:
459
102
            os.chdir(startdir)
460
103
        self.assertEqual(1, self.m__extract_root_layered_fsimage.call_count)
461
104
        self.assertEqual(0, self.m_download.call_count)
462
105
463
106
    def test_absolute_local_file_single(self):
464
107
        """extract_root_layered_fsimage_url supports absolute file:/// uris."""
465
108
        tmpd = self.tmp_dir()
466
109
        target = self.tmp_path("target_d", tmpd)
467
110
        fpath = self.tmp_path("my.img", tmpd)
468
111
        util.write_file(fpath, fpath + " data\n")
469
112
        extract_root_layered_fsimage_url("file:///" + fpath, target)
470
113
        self.assertEqual(1, self.m__extract_root_layered_fsimage.call_count)
471
114
        self.assertEqual(0, self.m_download.call_count)
472
115
473
116
    def test_local_file_path_single(self):
474
117
        """extract_root_layered_fsimage_url supports normal file path without
475
118
           file:"""
476
119
        tmpd = self.tmp_dir()
477
120
        target = self.tmp_path("target_d", tmpd)
478
121
        fpath = self.tmp_path("my.img", tmpd)
479
122
        util.write_file(fpath, fpath + " data\n")
480
123
        extract_root_layered_fsimage_url(os.path.abspath(fpath), target)
481
124
        self.assertEqual(1, self.m__extract_root_layered_fsimage.call_count)
482
125
        self.assertEqual(0, self.m_download.call_count)
483
126
484
127
    def test_local_file_path_multiple(self):
485
128
        """extract_root_layered_fsimage_url supports normal hierarchy file
486
129
           path"""
487
130
        tmpd = self.tmp_dir()
488
131
        target = self.tmp_path("target_d", tmpd)
489
132
        arg = os.path.abspath(self.tmp_path("minimal.standard.debug.squashfs",
490
133
                                            tmpd))
491
134
        for f in ["minimal.squashfs",
492
135
                  "minimal.standard.squashfs",
493
136
                  "minimal.standard.debug.squashfs"]:
494
137
            fpath = self.tmp_path(f, tmpd)
495
138
            util.write_file(fpath, fpath + " data\n")
496
139
        extract_root_layered_fsimage_url(arg, target)
497
140
        self.assertEqual(1, self.m__extract_root_layered_fsimage.call_count)
498
141
        self.assertEqual(0, self.m_download.call_count)
499
142
500
143
    def test_local_file_path_multiple_one_missing(self):
501
144
        """extract_root_layered_fsimage_url supports normal hierarchy file
502
145
           path but intermediate layer missing"""
503
146
        tmpd = self.tmp_dir()
504
147
        target = self.tmp_path("target_d", tmpd)
505
148
        arg = os.path.abspath(self.tmp_path("minimal.standard.debug.squashfs",
506
149
                                            tmpd))
507
150
        for f in ["minimal.squashfs",
508
151
                  "minimal.standard.debug.squashfs"]:
509
152
            fpath = self.tmp_path(f, tmpd)
510
153
            util.write_file(fpath, fpath + " data\n")
511
154
        self.assertRaises(ValueError, extract_root_layered_fsimage_url, arg,
512
155
                          target)
513
156
        self.assertEqual(0, self.m__extract_root_layered_fsimage.call_count)
514
157
        self.assertEqual(0, self.m_download.call_count)
515
158
516
159
    def test_local_file_path_multiple_one_empty(self):
517
160
        """extract_root_layered_fsimage_url supports normal hierarchy file
518
161
           path but intermediate layer empty"""
519
162
        tmpd = self.tmp_dir()
520
163
        target = self.tmp_path("target_d", tmpd)
521
164
        arg = os.path.abspath(self.tmp_path("minimal.standard.debug.squashfs",
522
165
                                            tmpd))
523
166
        for f in ["minimal.squashfs",
524
167
                  "minimal.standard.squashfs"
525
168
                  "minimal.standard.debug.squashfs"]:
526
169
            fpath = self.tmp_path(f, tmpd)
527
170
            if f == "minimal.standard.squashfs":
528
171
                util.write_file(fpath, "")
529
172
            else:
530
173
                util.write_file(fpath, fpath + " data\n")
531
174
        self.assertRaises(ValueError, extract_root_layered_fsimage_url, arg,
532
175
                          target)
533
176
        self.assertEqual(0, self.m__extract_root_layered_fsimage.call_count)
534
177
        self.assertEqual(0, self.m_download.call_count)
535
178
536
179
    def test_remote_file_single(self):
537
180
        """extract_root_layered_fsimage_url supports http:// urls."""
538
181
        tmpd = self.tmp_dir()
539
182
        target = self.tmp_path("target_d", tmpd)
540
183
        myurl = "http://example.io/minimal.squashfs"
541
184
        extract_root_layered_fsimage_url(myurl, target)
542
185
        self.assertEqual(1, self.m__extract_root_layered_fsimage.call_count)
543
186
        self.assertEqual(1, self.m_download.call_count)
544
187
        self.assertEqual("http://example.io/minimal.squashfs",
545
188
                         self.m_download.call_args_list[0][0][0])
546
189
        # ensure the file got cleaned up.
547
190
        self.assertEqual([], [f for f in self.downloads if os.path.exists(f)])
548
191
549
192
    def test_remote_file_multiple(self):
550
193
        """extract_root_layered_fsimage_url supports normal hierarchy from
551
194
           http:// urls."""
552
195
        tmpd = self.tmp_dir()
553
196
        target = self.tmp_path("target_d", tmpd)
554
197
        myurl = "http://example.io/minimal.standard.debug.squashfs"
555
198
        extract_root_layered_fsimage_url(myurl, target)
556
199
        self.assertEqual(1, self.m__extract_root_layered_fsimage.call_count)
557
200
        self.assertEqual(3, self.m_download.call_count)
558
201
        for i, image_url in enumerate(["minimal.squashfs",
559
202
                                       "minimal.standard.squashfs",
560
203
                                       "minimal.standard.debug.squashfs"]):
561
204
            self.assertEqual("http://example.io/" + image_url,
562
205
                             self.m_download.call_args_list[i][0][0])
563
206
        # ensure the file got cleaned up.
564
207
        self.assertEqual([], [f for f in self.downloads if os.path.exists(f)])
565
208
566
209
    def test_remote_file_multiple_one_missing(self):
567
210
        """extract_root_layered_fsimage_url supports normal hierarchy from
568
211
           http:// urls with one layer missing."""
569
212
570
213
        def fail_download_minimal_standard(url, path, retries=0):
571
214
            if url == "http://example.io/minimal.standard.squashfs":
572
215
                raise UrlError(url, 404, "Couldn't download",
573
216
                               None, None)
574
217
            return self._fake_download(url, path, retries)
575
218
        self.m_download.side_effect = fail_download_minimal_standard
576
219
577
220
        tmpd = self.tmp_dir()
578
221
        target = self.tmp_path("target_d", tmpd)
579
222
        myurl = "http://example.io/minimal.standard.debug.squashfs"
580
223
        self.assertRaises(UrlError, extract_root_layered_fsimage_url,
581
224
                          myurl, target)
582
225
        self.assertEqual(0, self.m__extract_root_layered_fsimage.call_count)
583
226
        self.assertEqual(2, self.m_download.call_count)
584
227
        for i, image_url in enumerate(["minimal.squashfs",
585
228
                                       "minimal.standard.squashfs"]):
586
229
            self.assertEqual("http://example.io/" + image_url,
587
230
                             self.m_download.call_args_list[i][0][0])
588
231
        # ensure the file got cleaned up.
589
232
        self.assertEqual([], [f for f in self.downloads if os.path.exists(f)])
590
233
591
234
    def test_remote_file_multiple_one_empty(self):
592
235
        """extract_root_layered_fsimage_url supports normal hierarchy from
593
236
           http:// urls with one layer empty."""
594
237
595
238
        def empty_download_minimal_standard(url, path, retries=0):
596
239
            if url == "http://example.io/minimal.standard.squashfs":
597
240
                self.downloads.append(os.path.abspath(path))
598
241
                with open(path, "w") as fp:
599
242
                    fp.write("")
600
243
                return
601
244
            return self._fake_download(url, path, retries)
602
245
        self.m_download.side_effect = empty_download_minimal_standard
603
246
604
247
        tmpd = self.tmp_dir()
605
248
        target = self.tmp_path("target_d", tmpd)
606
249
        myurl = "http://example.io/minimal.standard.debug.squashfs"
607
250
        self.assertRaises(ValueError, extract_root_layered_fsimage_url,
608
251
                          myurl, target)
609
252
        self.assertEqual(0, self.m__extract_root_layered_fsimage.call_count)
610
253
        self.assertEqual(3, self.m_download.call_count)
611
254
        for i, image_url in enumerate(["minimal.squashfs",
612
255
                                       "minimal.standard.squashfs",
613
256
                                       "minimal.standard.debug.squashfs"]):
614
257
            self.assertEqual("http://example.io/" + image_url,
615
258
                             self.m_download.call_args_list[i][0][0])
616
259
        # ensure the file got cleaned up.
617
260
        self.assertEqual([], [f for f in self.downloads if os.path.exists(f)])
618
261
619
262
620
263
class TestGetImageStack(CiTestCase):
621
264
    """Test _get_image_stack."""
622
265
623
266
    def test_get_image_stack(self):
624
267
        """_get_image_paths returns a tuple of depending fsimages
625
268
           with same extension"""
626
269
        self.assertEqual(
627
270
            ['/path/to/aa.fs',
628
271
             '/path/to/aa.bbb.fs',
629
272
             '/path/to/aa.bbb.cccc.fs'],
630
273
            _get_image_stack("/path/to/aa.bbb.cccc.fs"))
631
274
632
275
    def test_get_image_stack_none(self):
633
276
        """_get_image_paths returns an empty tuple with no entry"""
634
277
        self.assertEqual(
635
278
            [],
636
279
            _get_image_stack(""))
637
280
638
281
    def test_get_image_stack_no_dependency(self):
639
282
        """_get_image_paths returns a tuple a single element when fsimage
640
283
           has no dependency"""
641
284
        self.assertEqual(
642
285
            ['/path/to/aa.fs'],
643
286
            _get_image_stack("/path/to/aa.fs"))
644
287
645
288
    def test_get_image_stack_with_urls(self):
646
289
        """_get_image_paths returns a tuple of depending fsimages
647
290
           with same extension and same urls"""
648
291
        self.assertEqual(
649
292
            ['https://path.com/to/aa.fs',
650
293
             'https://path.com/to/aa.bbb.fs',
651
294
             'https://path.com/to/aa.bbb.cccc.fs'],
652
295
            _get_image_stack("https://path.com/to/aa.bbb.cccc.fs"))
653
296
654
73
# vi: ts=4 expandtab syntax=python
297
# vi: ts=4 expandtab syntax=python
655
diff --git a/tests/unittests/test_url_helper.py b/tests/unittests/test_url_helper.py
656
index 9cc9a5d..1b550f6 100644
657
--- a/tests/unittests/test_url_helper.py
658
+++ b/tests/unittests/test_url_helper.py
659
@@ -10,19 +10,71 @@ from .helpers import CiTestCase
660
10
10
661
11
11
662
12
class TestDownload(CiTestCase):
12
class TestDownload(CiTestCase):
663
13
    def setUp(self):
664
14
        super(TestDownload, self).setUp()
665
15
        self.tmpd = self.tmp_dir()
666
16
        self.src_file = self.tmp_path("my-source", self.tmpd)
667
17
        with open(self.src_file, "wb") as fp:
668
18
            # Write the min amount of bytes
669
19
            fp.write(b':-)\n' * int(8200/4))
670
20
        self.target_file = self.tmp_path("my-target", self.tmpd)
671
21
672
13
    def test_download_file_url(self):
22
    def test_download_file_url(self):
673
14
        """Download a file to another file."""
23
        """Download a file to another file."""
677
15
        tmpd = self.tmp_dir()
24
        url_helper.download("file://" + self.src_file, self.target_file)
678
16
        src_file = self.tmp_path("my-source", tmpd)
25
        self.assertTrue(filecmp.cmp(self.src_file, self.target_file),
679
17
        target_file = self.tmp_path("my-target", tmpd)
26
                        "Downloaded file differed from source file.")
680
27
681
28
    @mock.patch('curtin.url_helper.UrlReader')
682
29
    def test_download_file_url_retry(self, urlreader_mock):
683
30
        """Retry downloading a file with server error (http 5xx)."""
684
31
        urlreader_mock.side_effect = url_helper.UrlError(None, code=500)
685
32
686
33
        self.assertRaises(url_helper.UrlError, url_helper.download,
687
34
                          "file://" + self.src_file, self.target_file,
688
35
                          retries=3, retry_delay=0)
689
36
        self.assertEquals(4, urlreader_mock.call_count,
690
37
                          "Didn't call UrlReader 4 times (retries=3)")
691
38
692
39
    @mock.patch('curtin.url_helper.UrlReader')
693
40
    def test_download_file_url_no_retry(self, urlreader_mock):
694
41
        """No retry by default on downloading a file with server error
695
42
           (http 5xx)."""
696
43
        urlreader_mock.side_effect = url_helper.UrlError(None, code=500)
697
44
698
45
        self.assertRaises(url_helper.UrlError, url_helper.download,
699
46
                          "file://" + self.src_file, self.target_file,
700
47
                          retry_delay=0)
701
48
        self.assertEquals(1, urlreader_mock.call_count,
702
49
                          "Didn't call UrlReader once (retries=0)")
703
50
704
51
    @mock.patch('curtin.url_helper.UrlReader')
705
52
    def test_download_file_url_no_retry_on_client_error(self, urlreader_mock):
706
53
        """No retry by default on downloading a file with 4xx http error."""
707
54
        urlreader_mock.side_effect = url_helper.UrlError(None, code=404)
708
55
709
56
        self.assertRaises(url_helper.UrlError, url_helper.download,
710
57
                          "file://" + self.src_file, self.target_file,
711
58
                          retries=3, retry_delay=0)
712
59
        self.assertEquals(1, urlreader_mock.call_count,
713
60
                          "Didn't call UrlReader once (400 class error)")
714
18
61
719
19
        # Make sure we have > 8192 bytes in the file (buflen of UrlReader)
62
    def test_download_file_url_retry_then_success(self):
720
20
        with open(src_file, "wb") as fp:
63
        """Retry downloading a file with server error and then succeed."""
721
21
            for line in range(0, 1024):
64
        url_reader = url_helper.UrlReader
718
22
                fp.write(b'Who are the people in your neighborhood.\n')
722
23
65
725
24
        url_helper.download("file://" + src_file, target_file)
66
        with mock.patch('curtin.url_helper.UrlReader') as urlreader_mock:
726
25
        self.assertTrue(filecmp.cmp(src_file, target_file),
67
            # return first an error, then, real object
727
68
            def urlreader_download(url):
728
69
                urlreader_mock.side_effect = url_reader
729
70
                raise url_helper.UrlError(None, code=500)
730
71
            urlreader_mock.side_effect = urlreader_download
731
72
            url_helper.download("file://" + self.src_file, self.target_file,
732
73
                                retries=3, retry_delay=0)
733
74
        self.assertEquals(2, urlreader_mock.call_count,
734
75
                          "Didn't call UrlReader twice (first failing,"
735
76
                          "then success)")
736
77
        self.assertTrue(filecmp.cmp(self.src_file, self.target_file),
737
26
                        "Downloaded file differed from source file.")
78
                        "Downloaded file differed from source file.")
738
27
79
739
28
80
Status:	Merged
Approved by:	Ryan Harper on 2019-02-15
Approved revision:	a333c135a5d9192791697cec586562a4bb8be6df
Merge reported by:	Server Team CI bot
Merged at revision:	not available
Proposed branch:	~jibel/curtin/+git/add-layer-images-support:master
Merge into:	curtin:master
Diff against target:	737 lines (+544/-44) 8 files modified curtin/commands/extract.py (+119/-1) curtin/commands/install.py (+0/-4) curtin/url_helper.py (+36/-24) curtin/util.py (+5/-1) doc/topics/config.rst (+76/-2) tests/unittests/helpers.py (+20/-0) tests/unittests/test_commands_extract.py (+227/-3) tests/unittests/test_url_helper.py (+61/-9)
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
Server Team CI bot	continuous-integration		Approve on 2019-02-15
Ryan Harper (community)			Approve on 2019-02-15
Chad Smith			Approve on 2019-02-07
Scott Moser (community)		2018-11-28	Needs Information on 2018-12-18
Review via email: mp+359761@code.launchpad.net