1
=== modified file 'landscape/broker/registration.py'
2
--- landscape/broker/registration.py	2013-04-03 14:40:57 +0000
3
+++ landscape/broker/registration.py	2013-04-05 16:00:29 +0000
4
@@ -110,14 +110,14 @@
5
110
110
6
111
    def should_register(self):
111
    def should_register(self):
7
112
        id = self._identity
112
        id = self._identity
9
113
        # boolean logic is hard, I'm gonna use an if
113
        if id.secure_id:
10
114
            # We already have a secure ID, no need to register
11
115
            return False
12
114
        if self._config.cloud:
116
        if self._config.cloud:
15
115
            return bool(not id.secure_id
117
            return self._message_store.accepts("register-cloud-vm")
14
116
                        and self._message_store.accepts("register-cloud-vm"))
16
117
        elif self._config.provisioning_otp:
118
        elif self._config.provisioning_otp:
20
118
            return (not id.secure_id) and\
119
            return self._message_store.accepts("register-provisioned-machine")
21
119
                self._message_store.accepts("register-provisioned-machine")
120
        return bool(id.computer_title and id.account_name
19
120
        return bool(not id.secure_id and id.computer_title and id.account_name
22
121
                    and self._message_store.accepts("register"))
121
                    and self._message_store.accepts("register"))
23
122
122
24
123
    def register(self):
123
    def register(self):
25
124
124
26
=== modified file 'landscape/broker/store.py'
27
--- landscape/broker/store.py	2013-03-31 10:09:42 +0000
28
+++ landscape/broker/store.py	2013-04-05 16:00:29 +0000
29
@@ -1,4 +1,96 @@
31
1
"""Message storage."""
1
"""Message storage.
32
2
33
3
The sequencing system we use in the message store may be quite confusing
34
4
if you haven't looked at it in the last 10 minutes. For that reason, let's
35
5
review the mechanics here.
36
6
37
7
Our goal is to implement a reasonably robust system for delivering messages
38
8
from us to our peer. The system should be smart enough to recover if the peer
39
9
happens to lose messages that we have already sent, provided that these
40
10
messages are not too old (we'll see below what 'too old' means).
41
11
42
12
Messages added to the store are identified by increasing natural numbers, the
43
13
first message added is identified by 0, the second by 1, and so on. We call
44
14
"sequence" the number identifying the next message that we want to send. For
45
15
example, if the store has been added ten messages (that we represent with
46
16
uppercase letters) and we want start sending the first of them, our store
47
17
would like like::
48
18
49
19
    sequence: 0
50
20
    messages: A, B, C, D, E, F, G, H, I, J
51
21
              ^
52
22
53
23
The "^" marker is what we call "pending offset" and is the displacement of the
54
24
message we want to send next from the first message we have in the store.
55
25
56
26
Let's say we now send to our peer a batch of 3 sequential messages. In the
57
27
payload we include the body of the messages being sent and the sequence, which
58
28
identifies the first message of the batch. In this case the payload would look
59
29
like (pseudo-code)::
60
30
61
31
    (sequence: 0, messages: A, B, C)
62
32
63
33
If everything works fine on the other end, our peer replies with a payload that
64
34
would like::
65
35
66
36
    (next-expected-sequence: 4)
67
37
68
38
meaning that the peer as received all the three messages that we sent, an so
69
39
the next message it expects to receive is the one identified by the number 4.
70
40
At this point we update both our pending offset and our sequence values, and
71
41
the store now looks like::
72
42
73
43
    sequence: 4
74
44
    messages: A, B, C, D, E, F, G, H, I, J
75
45
                       ^
76
46
77
47
Great, now let's pretend that we send another batch, this time with five
78
48
messages::
79
49
80
50
    (sequence: 4, messages: D, E, F, G, H)
81
51
82
52
Our peer receives them fine responding with a payload looking like::
83
53
84
54
    (next-expected-sequence: 9)
85
55
86
56
meaning that it received all the eight messages we sent so far and it's waiting
87
57
for the ninth. This is the second successful batch that we send in a row, so we
88
58
can be reasonably confident that at least the messages in the first batch are
89
59
not really needed anymore. We delete them and we update our sequence and
90
60
pending offset accordingly::
91
61
92
62
    sequence: 9
93
63
    messages: D, E, F, G, H, I, J
94
64
                             ^
95
65
96
66
Note that we still want to keep around the messages we sent in the very last
97
67
batch, just in case. Indeed we now try to send a third batch with the last two
98
68
messages that we have, but our peer surprisingly replies us with this payload::
99
69
100
70
    (next-expected-sequence: 6)
101
71
102
72
Ouch! This means that something bad happened and our peer has somehow lost not
103
73
only the two messages that we sent in the last batch, but also the last three
104
74
messages of the former batch :(
105
75
106
76
Luckly we've kept enough old messages around that we can try to send them
107
77
again, we update our sequence and pending offset and the store looks like::
108
78
109
79
    sequence: 6
110
80
    messages: D, E, F, G, H, I, J
111
81
                    ^
112
82
113
83
We can now start again sending messages using the same strategy.
114
84
115
85
Note however that in the worst case scenario we could receive from our peer
116
86
a next-expected-sequence value which is so old to be outside our buffer
117
87
of already-sent messages. In that case there is now way we can recover the
118
88
lost messages, and we'll just send the oldest one that we have.
119
89
120
90
See L{MessageStore} for details about how messages are stored on the file
121
91
system and L{landscape.lib.message.got_next_expected} to check how the
122
92
strategy for updating the pending offset and the sequence is implemented.
123
93
"""
124
2
94
125
3
import time
95
import time
126
4
import itertools
96
import itertools
127
@@ -16,24 +108,15 @@
128
16
class MessageStore(object):
108
class MessageStore(object):
129
17
    """A message store which stores its messages in a file system hierarchy.
109
    """A message store which stores its messages in a file system hierarchy.
130
18
110
149
19
    The sequencing system we use in the message store may be quite
111
    Beside the "sequence" and the "pending offset" values described in the
150
20
    confusing if you haven't looked at it in the last 10 minutes.  For
112
    module docstring above, the L{MessageStore} also stores what we call
151
21
    that reason, let's review the terminology here.
113
    "server sequence", which is the next message number expected by the
152
22
114
    *client* itself (because we are in turn the peer of a specular message
153
23
    Assume we have 10 messages in the store, which we label by
115
    system running in the server, which tries to deliver messages to us).
154
24
    the following uppercase letters::
116
155
25
117
    The server sequence is entirely unrelated to the stored messages, but is
156
26
        A, B, C, D, E, F, G, H, I, J
118
    incremented when successfully receiving messages from the server, in the
157
27
                 ^
119
    very same way described above but with the roles inverted.
140
28
141
29
    Let's say that the next message we should send to the server is D.
142
30
    What we call "pending offset" is the displacement from the first
143
31
    message, which in our example above would be 3.  What we call
144
32
    "sequence" is the number that the server expects us to label message
145
33
    D as.  It could be pretty much any natural number, depending on the
146
34
    history of our exchanges with the server.  What we call "server
147
35
    sequence", is the next message number expected by the *client* itself,
148
36
    and is entirely unrelated to the stored messages.
158
37
120
159
38
    @param persist: a L{Persist} used to save state parameters like the
121
    @param persist: a L{Persist} used to save state parameters like the
160
39
        accepted message types, sequence, server uuid etc.
122
        accepted message types, sequence, server uuid etc.
161
40
123
162
=== modified file 'landscape/lib/amp.py'
163
--- landscape/lib/amp.py	2013-03-27 22:58:20 +0000
164
+++ landscape/lib/amp.py	2013-04-05 16:00:29 +0000
165
@@ -4,12 +4,8 @@
166
4
from twisted.internet.protocol import ReconnectingClientFactory
4
from twisted.internet.protocol import ReconnectingClientFactory
167
5
from twisted.python.failure import Failure
5
from twisted.python.failure import Failure
168
6
6
175
7
try:
7
from twisted.protocols.amp import (
176
8
    from twisted.protocols.amp import (
8
    Argument, String, Integer, Command, AMP, MAX_VALUE_LENGTH)
171
9
        Argument, String, Integer, Command, AMP, MAX_VALUE_LENGTH)
172
10
except ImportError:
173
11
    from landscape.lib.twisted_amp import (
174
12
        Argument, String, Integer, Command, AMP, MAX_VALUE_LENGTH)
177
13
9
178
14
from landscape.lib.bpickle import loads, dumps, dumps_table
10
from landscape.lib.bpickle import loads, dumps, dumps_table
179
15
11
180
16
12
181
=== removed file 'landscape/lib/configobj.py'
182
--- landscape/lib/configobj.py	2010-04-28 13:07:53 +0000
183
+++ landscape/lib/configobj.py	1970-01-01 00:00:00 +0000
184
@@ -1,3728 +0,0 @@
185
1
# configobj.py
186
2
# A config file reader/writer that supports nested sections in config files.
187
3
# Copyright (C) 2005-2006 Michael Foord, Nicola Larosa
188
4
# E-mail: fuzzyman AT voidspace DOT org DOT uk
189
5
#         nico AT tekNico DOT net
190
6
191
7
# ConfigObj 4
192
8
# http://www.voidspace.org.uk/python/configobj.html
193
9
194
10
# Released subject to the BSD License
195
11
# Please see http://www.voidspace.org.uk/python/license.shtml
196
12
197
13
# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
198
14
# For information about bugfixes, updates and support, please join the
199
15
# ConfigObj mailing list:
200
16
# http://lists.sourceforge.net/lists/listinfo/configobj-develop
201
17
# Comments, suggestions and bug reports welcome.
202
18
203
19
from __future__ import generators
204
20
205
21
"""
206
22
    >>> z = ConfigObj()
207
23
    >>> z['a'] = 'a'
208
24
    >>> z['sect'] = {
209
25
    ...    'subsect': {
210
26
    ...         'a': 'fish',
211
27
    ...         'b': 'wobble',
212
28
    ...     },
213
29
    ...     'member': 'value',
214
30
    ... }
215
31
    >>> x = ConfigObj(z.write())
216
32
    >>> z == x
217
33
    1
218
34
"""
219
35
220
36
import sys
221
37
INTP_VER = sys.version_info[:2]
222
38
if INTP_VER < (2, 2):
223
39
    raise RuntimeError("Python v.2.2 or later needed")
224
40
225
41
import os
226
42
import re
227
43
import compiler
228
44
from types import StringTypes
229
45
from warnings import warn
230
46
from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
231
47
232
48
# A dictionary mapping BOM to
233
49
# the encoding to decode with, and what to set the
234
50
# encoding attribute to.
235
51
BOMS = {
236
52
    BOM_UTF8: ('utf_8', None),
237
53
    BOM_UTF16_BE: ('utf16_be', 'utf_16'),
238
54
    BOM_UTF16_LE: ('utf16_le', 'utf_16'),
239
55
    BOM_UTF16: ('utf_16', 'utf_16'),
240
56
    }
241
57
# All legal variants of the BOM codecs.
242
58
# TODO: the list of aliases is not meant to be exhaustive, is there a
243
59
#   better way ?
244
60
BOM_LIST = {
245
61
    'utf_16': 'utf_16',
246
62
    'u16': 'utf_16',
247
63
    'utf16': 'utf_16',
248
64
    'utf-16': 'utf_16',
249
65
    'utf16_be': 'utf16_be',
250
66
    'utf_16_be': 'utf16_be',
251
67
    'utf-16be': 'utf16_be',
252
68
    'utf16_le': 'utf16_le',
253
69
    'utf_16_le': 'utf16_le',
254
70
    'utf-16le': 'utf16_le',
255
71
    'utf_8': 'utf_8',
256
72
    'u8': 'utf_8',
257
73
    'utf': 'utf_8',
258
74
    'utf8': 'utf_8',
259
75
    'utf-8': 'utf_8',
260
76
    }
261
77
262
78
# Map of encodings to the BOM to write.
263
79
BOM_SET = {
264
80
    'utf_8': BOM_UTF8,
265
81
    'utf_16': BOM_UTF16,
266
82
    'utf16_be': BOM_UTF16_BE,
267
83
    'utf16_le': BOM_UTF16_LE,
268
84
    None: BOM_UTF8}
269
85
270
86
try:
271
87
    from validate import VdtMissingValue
272
88
except ImportError:
273
89
    VdtMissingValue = None
274
90
275
91
try:
276
92
    enumerate
277
93
except NameError:
278
94
279
95
    def enumerate(obj):
280
96
        """enumerate for Python 2.2."""
281
97
        i = -1
282
98
        for item in obj:
283
99
            i += 1
284
100
            yield i, item
285
101
286
102
try:
287
103
    True, False
288
104
except NameError:
289
105
    True, False = 1, 0
290
106
291
107
292
108
__version__ = '4.3.0alpha2'
293
109
294
110
__revision__ = '$Id: configobj.py 156 2006-01-31 14:57:08Z fuzzyman $'
295
111
296
112
__docformat__ = "restructuredtext en"
297
113
298
114
# NOTE: Does it make sense to have the following in __all__ ?
299
115
# NOTE: DEFAULT_INDENT_TYPE, NUM_INDENT_SPACES, MAX_INTERPOL_DEPTH
300
116
# NOTE: If used via ``from configobj import...``
301
117
# NOTE: They are effectively read only
302
118
__all__ = (
303
119
    '__version__',
304
120
    'DEFAULT_INDENT_TYPE',
305
121
    'NUM_INDENT_SPACES',
306
122
    'MAX_INTERPOL_DEPTH',
307
123
    'ConfigObjError',
308
124
    'NestingError',
309
125
    'ParseError',
310
126
    'DuplicateError',
311
127
    'ConfigspecError',
312
128
    'ConfigObj',
313
129
    'SimpleVal',
314
130
    'InterpolationError',
315
131
    'InterpolationDepthError',
316
132
    'MissingInterpolationOption',
317
133
    'RepeatSectionError',
318
134
    '__docformat__',
319
135
    'flatten_errors',
320
136
)
321
137
322
138
DEFAULT_INDENT_TYPE = ' '
323
139
NUM_INDENT_SPACES = 4
324
140
MAX_INTERPOL_DEPTH = 10
325
141
326
142
OPTION_DEFAULTS = {
327
143
    'interpolation': True,
328
144
    'raise_errors': False,
329
145
    'list_values': True,
330
146
    'create_empty': False,
331
147
    'file_error': False,
332
148
    'configspec': None,
333
149
    'stringify': True,
334
150
    # option may be set to one of ('', ' ', '\t')
335
151
    'indent_type': None,
336
152
    'encoding': None,
337
153
    'default_encoding': None,
338
154
    'unrepr': False,
339
155
    'write_empty_values': False,
340
156
}
341
157
342
158
343
159
def getObj(s):
344
160
    s = "a=" + s
345
161
    p = compiler.parse(s)
346
162
    return p.getChildren()[1].getChildren()[0].getChildren()[1]
347
163
348
164
349
165
class UnknownType(Exception):
350
166
    pass
351
167
352
168
353
169
class Builder:
354
170
355
171
    def build(self, o):
356
172
        m = getattr(self, 'build_' + o.__class__.__name__, None)
357
173
        if m is None:
358
174
            raise UnknownType(o.__class__.__name__)
359
175
        return m(o)
360
176
361
177
    def build_List(self, o):
362
178
        return map(self.build, o.getChildren())
363
179
364
180
    def build_Const(self, o):
365
181
        return o.value
366
182
367
183
    def build_Dict(self, o):
368
184
        d = {}
369
185
        i = iter(map(self.build, o.getChildren()))
370
186
        for el in i:
371
187
            d[el] = i.next()
372
188
        return d
373
189
374
190
    def build_Tuple(self, o):
375
191
        return tuple(self.build_List(o))
376
192
377
193
    def build_Name(self, o):
378
194
        if o.name == 'None':
379
195
            return None
380
196
        if o.name == 'True':
381
197
            return True
382
198
        if o.name == 'False':
383
199
            return False
384
200
385
201
        # An undefinted Name
386
202
        raise UnknownType('Undefined Name')
387
203
388
204
    def build_Add(self, o):
389
205
        real, imag = map(self.build_Const, o.getChildren())
390
206
        try:
391
207
            real = float(real)
392
208
        except TypeError:
393
209
            raise UnknownType('Add')
394
210
        if not isinstance(imag, complex) or imag.real != 0.0:
395
211
            raise UnknownType('Add')
396
212
        return real + imag
397
213
398
214
    def build_Getattr(self, o):
399
215
        parent = self.build(o.expr)
400
216
        return getattr(parent, o.attrname)
401
217
402
218
403
219
def unrepr(s):
404
220
    if not s:
405
221
        return s
406
222
    return Builder().build(getObj(s))
407
223
408
224
409
225
class ConfigObjError(SyntaxError):
410
226
    """
411
227
    This is the base class for all errors that ConfigObj raises.
412
228
    It is a subclass of SyntaxError.
413
229
414
230
    >>> raise ConfigObjError
415
231
    Traceback (most recent call last):
416
232
    ConfigObjError
417
233
    """
418
234
419
235
    def __init__(self, message='', line_number=None, line=''):
420
236
        self.line = line
421
237
        self.line_number = line_number
422
238
        self.message = message
423
239
        SyntaxError.__init__(self, message)
424
240
425
241
426
242
class NestingError(ConfigObjError):
427
243
    """
428
244
    This error indicates a level of nesting that doesn't match.
429
245
430
246
    >>> raise NestingError
431
247
    Traceback (most recent call last):
432
248
    NestingError
433
249
    """
434
250
435
251
436
252
class ParseError(ConfigObjError):
437
253
    """
438
254
    This error indicates that a line is badly written.
439
255
    It is neither a valid ``key = value`` line,
440
256
    nor a valid section marker line.
441
257
442
258
    >>> raise ParseError
443
259
    Traceback (most recent call last):
444
260
    ParseError
445
261
    """
446
262
447
263
448
264
class DuplicateError(ConfigObjError):
449
265
    """
450
266
    The keyword or section specified already exists.
451
267
452
268
    >>> raise DuplicateError
453
269
    Traceback (most recent call last):
454
270
    DuplicateError
455
271
    """
456
272
457
273
458
274
class ConfigspecError(ConfigObjError):
459
275
    """
460
276
    An error occured whilst parsing a configspec.
461
277
462
278
    >>> raise ConfigspecError
463
279
    Traceback (most recent call last):
464
280
    ConfigspecError
465
281
    """
466
282
467
283
468
284
class InterpolationError(ConfigObjError):
469
285
    """Base class for the two interpolation errors."""
470
286
471
287
472
288
class InterpolationDepthError(InterpolationError):
473
289
    """Maximum interpolation depth exceeded in string interpolation."""
474
290
475
291
    def __init__(self, option):
476
292
        """
477
293
        >>> raise InterpolationDepthError('yoda')
478
294
        Traceback (most recent call last):
479
295
        InterpolationDepthError: max interpolation depth exceeded in value "yoda".
480
296
        """
481
297
        InterpolationError.__init__(
482
298
            self,
483
299
            'max interpolation depth exceeded in value "%s".' % option)
484
300
485
301
486
302
class RepeatSectionError(ConfigObjError):
487
303
    """
488
304
    This error indicates additional sections in a section with a
489
305
    ``__many__`` (repeated) section.
490
306
491
307
    >>> raise RepeatSectionError
492
308
    Traceback (most recent call last):
493
309
    RepeatSectionError
494
310
    """
495
311
496
312
497
313
class MissingInterpolationOption(InterpolationError):
498
314
    """A value specified for interpolation was missing."""
499
315
500
316
    def __init__(self, option):
501
317
        """
502
318
        >>> raise MissingInterpolationOption('yoda')
503
319
        Traceback (most recent call last):
504
320
        MissingInterpolationOption: missing option "yoda" in interpolation.
505
321
        """
506
322
        InterpolationError.__init__(
507
323
            self,
508
324
            'missing option "%s" in interpolation.' % option)
509
325
510
326
511
327
class Section(dict):
512
328
    """
513
329
    A dictionary-like object that represents a section in a config file.
514
330
515
331
    It does string interpolation if the 'interpolate' attribute
516
332
    of the 'main' object is set to True.
517
333
518
334
    Interpolation is tried first from the 'DEFAULT' section of this object,
519
335
    next from the 'DEFAULT' section of the parent, lastly the main object.
520
336
521
337
    A Section will behave like an ordered dictionary - following the
522
338
    order of the ``scalars`` and ``sections`` attributes.
523
339
    You can use this to change the order of members.
524
340
525
341
    Iteration follows the order: scalars, then sections.
526
342
    """
527
343
528
344
    _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
529
345
530
346
    def __init__(self, parent, depth, main, indict=None, name=None):
531
347
        """
532
348
        * parent is the section above
533
349
        * depth is the depth level of this section
534
350
        * main is the main ConfigObj
535
351
        * indict is a dictionary to initialise the section with
536
352
        """
537
353
        if indict is None:
538
354
            indict = {}
539
355
        dict.__init__(self)
540
356
        # used for nesting level *and* interpolation
541
357
        self.parent = parent
542
358
        # used for the interpolation attribute
543
359
        self.main = main
544
360
        # level of nesting depth of this Section
545
361
        self.depth = depth
546
362
        # the sequence of scalar values in this Section
547
363
        self.scalars = []
548
364
        # the sequence of sections in this Section
549
365
        self.sections = []
550
366
        # purely for information
551
367
        self.name = name
552
368
        # for comments :-)
553
369
        self.comments = {}
554
370
        self.inline_comments = {}
555
371
        # for the configspec
556
372
        self.configspec = {}
557
373
        self._order = []
558
374
        self._configspec_comments = {}
559
375
        self._configspec_inline_comments = {}
560
376
        self._cs_section_comments = {}
561
377
        self._cs_section_inline_comments = {}
562
378
        # for defaults
563
379
        self.defaults = []
564
380
        #
565
381
        # we do this explicitly so that __setitem__ is used properly
566
382
        # (rather than just passing to ``dict.__init__``)
567
383
        for entry in indict:
568
384
            self[entry] = indict[entry]
569
385
570
386
    def _interpolate(self, value):
571
387
        """Nicked from ConfigParser."""
572
388
        depth = MAX_INTERPOL_DEPTH
573
389
        # loop through this until it's done
574
390
        while depth:
575
391
            depth -= 1
576
392
            if value.find("%(") != -1:
577
393
                value = self._KEYCRE.sub(self._interpolation_replace, value)
578
394
            else:
579
395
                break
580
396
        else:
581
397
            raise InterpolationDepthError(value)
582
398
        return value
583
399
584
400
    def _interpolation_replace(self, match):
585
401
        """ """
586
402
        s = match.group(1)
587
403
        if s is None:
588
404
            return match.group()
589
405
        else:
590
406
            # switch off interpolation before we try and fetch anything !
591
407
            self.main.interpolation = False
592
408
            # try the 'DEFAULT' member of *this section* first
593
409
            val = self.get('DEFAULT', {}).get(s)
594
410
            # try the 'DEFAULT' member of the *parent section* next
595
411
            if val is None:
596
412
                val = self.parent.get('DEFAULT', {}).get(s)
597
413
            # last, try the 'DEFAULT' member of the *main section*
598
414
            if val is None:
599
415
                val = self.main.get('DEFAULT', {}).get(s)
600
416
            self.main.interpolation = True
601
417
            if val is None:
602
418
                raise MissingInterpolationOption(s)
603
419
            return val
604
420
605
421
    def __getitem__(self, key):
606
422
        """Fetch the item and do string interpolation."""
607
423
        val = dict.__getitem__(self, key)
608
424
        if self.main.interpolation and isinstance(val, StringTypes):
609
425
            return self._interpolate(val)
610
426
        return val
611
427
612
428
    def __setitem__(self, key, value, unrepr=False):
613
429
        """
614
430
        Correctly set a value.
615
431
616
432
        Making dictionary values Section instances.
617
433
        (We have to special case 'Section' instances - which are also dicts)
618
434
619
435
        Keys must be strings.
620
436
        Values need only be strings (or lists of strings) if
621
437
        ``main.stringify`` is set.
622
438
623
439
        `unrepr`` must be set when setting a value to a dictionary, without
624
440
        creating a new sub-section.
625
441
        """
626
442
        if not isinstance(key, StringTypes):
627
443
            raise ValueError, 'The key "%s" is not a string.' % key
628
444
        # add the comment
629
445
        if not key in self.comments:
630
446
            self.comments[key] = []
631
447
            self.inline_comments[key] = ''
632
448
        # remove the entry from defaults
633
449
        if key in self.defaults:
634
450
            self.defaults.remove(key)
635
451
        #
636
452
        if isinstance(value, Section):
637
453
            if not self.has_key(key):
638
454
                self.sections.append(key)
639
455
            dict.__setitem__(self, key, value)
640
456
        elif isinstance(value, dict)and not unrepr:
641
457
            # First create the new depth level,
642
458
            # then create the section
643
459
            if not self.has_key(key):
644
460
                self.sections.append(key)
645
461
            new_depth = self.depth + 1
646
462
            dict.__setitem__(
647
463
                self,
648
464
                key,
649
465
                Section(
650
466
                    self,
651
467
                    new_depth,
652
468
                    self.main,
653
469
                    indict=value,
654
470
                    name=key))
655
471
        else:
656
472
            if not self.has_key(key):
657
473
                self.scalars.append(key)
658
474
            if not self.main.stringify:
659
475
                if isinstance(value, StringTypes):
660
476
                    pass
661
477
                elif isinstance(value, (list, tuple)):
662
478
                    for entry in value:
663
479
                        if not isinstance(entry, StringTypes):
664
480
                            raise TypeError, (
665
481
                                'Value is not a string "%s".' % entry)
666
482
                else:
667
483
                    raise TypeError, 'Value is not a string "%s".' % value
668
484
            dict.__setitem__(self, key, value)
669
485
670
486
    def __delitem__(self, key):
671
487
        """Remove items from the sequence when deleting."""
672
488
        dict. __delitem__(self, key)
673
489
        if key in self.scalars:
674
490
            self.scalars.remove(key)
675
491
        else:
676
492
            self.sections.remove(key)
677
493
        del self.comments[key]
678
494
        del self.inline_comments[key]
679
495
680
496
    def get(self, key, default=None):
681
497
        """A version of ``get`` that doesn't bypass string interpolation."""
682
498
        try:
683
499
            return self[key]
684
500
        except KeyError:
685
501
            return default
686
502
687
503
    def update(self, indict):
688
504
        """
689
505
        A version of update that uses our ``__setitem__``.
690
506
        """
691
507
        for entry in indict:
692
508
            self[entry] = indict[entry]
693
509
694
510
    def pop(self, key, *args):
695
511
        """ """
696
512
        val = dict.pop(self, key, *args)
697
513
        if key in self.scalars:
698
514
            del self.comments[key]
699
515
            del self.inline_comments[key]
700
516
            self.scalars.remove(key)
701
517
        elif key in self.sections:
702
518
            del self.comments[key]
703
519
            del self.inline_comments[key]
704
520
            self.sections.remove(key)
705
521
        if self.main.interpolation and isinstance(val, StringTypes):
706
522
            return self._interpolate(val)
707
523
        return val
708
524
709
525
    def popitem(self):
710
526
        """Pops the first (key,val)"""
711
527
        sequence = (self.scalars + self.sections)
712
528
        if not sequence:
713
529
            raise KeyError, ": 'popitem(): dictionary is empty'"
714
530
        key = sequence[0]
715
531
        val = self[key]
716
532
        del self[key]
717
533
        return key, val
718
534
719
535
    def clear(self):
720
536
        """
721
537
        A version of clear that also affects scalars/sections
722
538
        Also clears comments and configspec.
723
539
724
540
        Leaves other attributes alone :
725
541
            depth/main/parent are not affected
726
542
        """
727
543
        dict.clear(self)
728
544
        self.scalars = []
729
545
        self.sections = []
730
546
        self.comments = {}
731
547
        self.inline_comments = {}
732
548
        self.configspec = {}
733
549
734
550
    def setdefault(self, key, default=None):
735
551
        """A version of setdefault that sets sequence if appropriate."""
736
552
        try:
737
553
            return self[key]
738
554
        except KeyError:
739
555
            self[key] = default
740
556
            return self[key]
741
557
742
558
    def items(self):
743
559
        """ """
744
560
        return zip((self.scalars + self.sections), self.values())
745
561
746
562
    def keys(self):
747
563
        """ """
748
564
        return (self.scalars + self.sections)
749
565
750
566
    def values(self):
751
567
        """ """
752
568
        return [self[key] for key in (self.scalars + self.sections)]
753
569
754
570
    def iteritems(self):
755
571
        """ """
756
572
        return iter(self.items())
757
573
758
574
    def iterkeys(self):
759
575
        """ """
760
576
        return iter((self.scalars + self.sections))
761
577
762
578
    __iter__ = iterkeys
763
579
764
580
    def itervalues(self):
765
581
        """ """
766
582
        return iter(self.values())
767
583
768
584
    def __repr__(self):
769
585
        return '{%s}' % ', '.join([('%s: %s' % (repr(key), repr(self[key])))
770
586
            for key in (self.scalars + self.sections)])
771
587
772
588
    __str__ = __repr__
773
589
774
590
    # Extra methods - not in a normal dictionary
775
591
776
592
    def dict(self):
777
593
        """
778
594
        Return a deepcopy of self as a dictionary.
779
595
780
596
        All members that are ``Section`` instances are recursively turned to
781
597
        ordinary dictionaries - by calling their ``dict`` method.
782
598
783
599
        >>> n = a.dict()
784
600
        >>> n == a
785
601
        1
786
602
        >>> n is a
787
603
        0
788
604
        """
789
605
        newdict = {}
790
606
        for entry in self:
791
607
            this_entry = self[entry]
792
608
            if isinstance(this_entry, Section):
793
609
                this_entry = this_entry.dict()
794
610
            # XXX Modified to return tuples as tuples. -- niemeyer, 2006-04-24
795
611
            elif isinstance(this_entry, list):
796
612
                this_entry = list(this_entry)
797
613
            elif isinstance(this_entry, tuple):
798
614
                this_entry = tuple(this_entry)
799
615
            newdict[entry] = this_entry
800
616
        return newdict
801
617
802
618
    def merge(self, indict):
803
619
        """
804
620
        A recursive update - useful for merging config files.
805
621
806
622
        >>> a = '''[section1]
807
623
        ...     option1 = True
808
624
        ...     [[subsection]]
809
625
        ...     more_options = False
810
626
        ...     # end of file'''.splitlines()
811
627
        >>> b = '''# File is user.ini
812
628
        ...     [section1]
813
629
        ...     option1 = False
814
630
        ...     # end of file'''.splitlines()
815
631
        >>> c1 = ConfigObj(b)
816
632
        >>> c2 = ConfigObj(a)
817
633
        >>> c2.merge(c1)
818
634
        >>> c2
819
635
        {'section1': {'option1': 'False', 'subsection': {'more_options': 'False'}}}
820
636
        """
821
637
        for key, val in indict.items():
822
638
            if (key in self and isinstance(self[key], dict) and
823
639
                                isinstance(val, dict)):
824
640
                self[key].merge(val)
825
641
            else:
826
642
                self[key] = val
827
643
828
644
    def rename(self, oldkey, newkey):
829
645
        """
830
646
        Change a keyname to another, without changing position in sequence.
831
647
832
648
        Implemented so that transformations can be made on keys,
833
649
        as well as on values. (used by encode and decode)
834
650
835
651
        Also renames comments.
836
652
        """
837
653
        if oldkey in self.scalars:
838
654
            the_list = self.scalars
839
655
        elif oldkey in self.sections:
840
656
            the_list = self.sections
841
657
        else:
842
658
            raise KeyError, 'Key "%s" not found.' % oldkey
843
659
        pos = the_list.index(oldkey)
844
660
        #
845
661
        val = self[oldkey]
846
662
        dict.__delitem__(self, oldkey)
847
663
        dict.__setitem__(self, newkey, val)
848
664
        the_list.remove(oldkey)
849
665
        the_list.insert(pos, newkey)
850
666
        comm = self.comments[oldkey]
851
667
        inline_comment = self.inline_comments[oldkey]
852
668
        del self.comments[oldkey]
853
669
        del self.inline_comments[oldkey]
854
670
        self.comments[newkey] = comm
855
671
        self.inline_comments[newkey] = inline_comment
856
672
857
673
    def walk(self, function, raise_errors=True,
858
674
            call_on_sections=False, **keywargs):
859
675
        """
860
676
        Walk every member and call a function on the keyword and value.
861
677
862
678
        Return a dictionary of the return values
863
679
864
680
        If the function raises an exception, raise the errror
865
681
        unless ``raise_errors=False``, in which case set the return value to
866
682
        ``False``.
867
683
868
684
        Any unrecognised keyword arguments you pass to walk, will be pased on
869
685
        to the function you pass in.
870
686
871
687
        Note: if ``call_on_sections`` is ``True`` then - on encountering a
872
688
        subsection, *first* the function is called for the *whole* subsection,
873
689
        and then recurses into it's members. This means your function must be
874
690
        able to handle strings, dictionaries and lists. This allows you
875
691
        to change the key of subsections as well as for ordinary members. The
876
692
        return value when called on the whole subsection has to be discarded.
877
693
878
694
        See  the encode and decode methods for examples, including functions.
879
695
880
696
        .. caution::
881
697
882
698
            You can use ``walk`` to transform the names of members of a section
883
699
            but you mustn't add or delete members.
884
700
885
701
        >>> config = '''[XXXXsection]
886
702
        ... XXXXkey = XXXXvalue'''.splitlines()
887
703
        >>> cfg = ConfigObj(config)
888
704
        >>> cfg
889
705
        {'XXXXsection': {'XXXXkey': 'XXXXvalue'}}
890
706
        >>> def transform(section, key):
891
707
        ...     val = section[key]
892
708
        ...     newkey = key.replace('XXXX', 'CLIENT1')
893
709
        ...     section.rename(key, newkey)
894
710
        ...     if isinstance(val, (tuple, list, dict)):
895
711
        ...         pass
896
712
        ...     else:
897
713
        ...         val = val.replace('XXXX', 'CLIENT1')
898
714
        ...         section[newkey] = val
899
715
        >>> cfg.walk(transform, call_on_sections=True)
900
716
        {'CLIENT1section': {'CLIENT1key': None}}
901
717
        >>> cfg
902
718
        {'CLIENT1section': {'CLIENT1key': 'CLIENT1value'}}
903
719
        """
904
720
        out = {}
905
721
        # scalars first
906
722
        for i in range(len(self.scalars)):
907
723
            entry = self.scalars[i]
908
724
            try:
909
725
                val = function(self, entry, **keywargs)
910
726
                # bound again in case name has changed
911
727
                entry = self.scalars[i]
912
728
                out[entry] = val
913
729
            except Exception:
914
730
                if raise_errors:
915
731
                    raise
916
732
                else:
917
733
                    entry = self.scalars[i]
918
734
                    out[entry] = False
919
735
        # then sections
920
736
        for i in range(len(self.sections)):
921
737
            entry = self.sections[i]
922
738
            if call_on_sections:
923
739
                try:
924
740
                    function(self, entry, **keywargs)
925
741
                except Exception:
926
742
                    if raise_errors:
927
743
                        raise
928
744
                    else:
929
745
                        entry = self.sections[i]
930
746
                        out[entry] = False
931
747
                # bound again in case name has changed
932
748
                entry = self.sections[i]
933
749
            # previous result is discarded
934
750
            out[entry] = self[entry].walk(
935
751
                function,
936
752
                raise_errors=raise_errors,
937
753
                call_on_sections=call_on_sections,
938
754
                **keywargs)
939
755
        return out
940
756
941
757
    def decode(self, encoding):
942
758
        """
943
759
        Decode all strings and values to unicode, using the specified encoding.
944
760
945
761
        Works with subsections and list values.
946
762
947
763
        Uses the ``walk`` method.
948
764
949
765
        Testing ``encode`` and ``decode``.
950
766
        >>> m = ConfigObj(a)
951
767
        >>> m.decode('ascii')
952
768
        >>> def testuni(val):
953
769
        ...     for entry in val:
954
770
        ...         if not isinstance(entry, unicode):
955
771
        ...             print >> sys.stderr, type(entry)
956
772
        ...             raise AssertionError, 'decode failed.'
957
773
        ...         if isinstance(val[entry], dict):
958
774
        ...             testuni(val[entry])
959
775
        ...         elif not isinstance(val[entry], unicode):
960
776
        ...             raise AssertionError, 'decode failed.'
961
777
        >>> testuni(m)
962
778
        >>> m.encode('ascii')
963
779
        >>> a == m
964
780
        1
965
781
        """
966
782
        warn('use of ``decode`` is deprecated.', DeprecationWarning)
967
783
968
784
        def decode(section, key, encoding=encoding, warn=True):
969
785
            """ """
970
786
            val = section[key]
971
787
            if isinstance(val, (list, tuple)):
972
788
                newval = []
973
789
                for entry in val:
974
790
                    newval.append(entry.decode(encoding))
975
791
            elif isinstance(val, dict):
976
792
                newval = val
977
793
            else:
978
794
                newval = val.decode(encoding)
979
795
            newkey = key.decode(encoding)
980
796
            section.rename(key, newkey)
981
797
            section[newkey] = newval
982
798
        # using ``call_on_sections`` allows us to modify section names
983
799
        self.walk(decode, call_on_sections=True)
984
800
985
801
    def encode(self, encoding):
986
802
        """
987
803
        Encode all strings and values from unicode,
988
804
        using the specified encoding.
989
805
990
806
        Works with subsections and list values.
991
807
        Uses the ``walk`` method.
992
808
        """
993
809
        warn('use of ``encode`` is deprecated.', DeprecationWarning)
994
810
        def encode(section, key, encoding=encoding):
995
811
            """ """
996
812
            val = section[key]
997
813
            if isinstance(val, (list, tuple)):
998
814
                newval = []
999
815
                for entry in val:
1000
816
                    newval.append(entry.encode(encoding))
1001
817
            elif isinstance(val, dict):
1002
818
                newval = val
1003
819
            else:
1004
820
                newval = val.encode(encoding)
1005
821
            newkey = key.encode(encoding)
1006
822
            section.rename(key, newkey)
1007
823
            section[newkey] = newval
1008
824
        self.walk(encode, call_on_sections=True)
1009
825
1010
826
    def istrue(self, key):
1011
827
        """A deprecated version of ``as_bool``."""
1012
828
        warn('use of ``istrue`` is deprecated. Use ``as_bool`` method '
1013
829
                'instead.', DeprecationWarning)
1014
830
        return self.as_bool(key)
1015
831
1016
832
    def as_bool(self, key):
1017
833
        """
1018
834
        Accepts a key as input. The corresponding value must be a string or
1019
835
        the objects (``True`` or 1) or (``False`` or 0). We allow 0 and 1 to
1020
836
        retain compatibility with Python 2.2.
1021
837
1022
838
        If the string is one of  ``True``, ``On``, ``Yes``, or ``1`` it returns
1023
839
        ``True``.
1024
840
1025
841
        If the string is one of  ``False``, ``Off``, ``No``, or ``0`` it returns
1026
842
        ``False``.
1027
843
1028
844
        ``as_bool`` is not case sensitive.
1029
845
1030
846
        Any other input will raise a ``ValueError``.
1031
847
1032
848
        >>> a = ConfigObj()
1033
849
        >>> a['a'] = 'fish'
1034
850
        >>> a.as_bool('a')
1035
851
        Traceback (most recent call last):
1036
852
        ValueError: Value "fish" is neither True nor False
1037
853
        >>> a['b'] = 'True'
1038
854
        >>> a.as_bool('b')
1039
855
        1
1040
856
        >>> a['b'] = 'off'
1041
857
        >>> a.as_bool('b')
1042
858
        0
1043
859
        """
1044
860
        val = self[key]
1045
861
        if val == True:
1046
862
            return True
1047
863
        elif val == False:
1048
864
            return False
1049
865
        else:
1050
866
            try:
1051
867
                if not isinstance(val, StringTypes):
1052
868
                    raise KeyError
1053
869
                else:
1054
870
                    return self.main._bools[val.lower()]
1055
871
            except KeyError:
1056
872
                raise ValueError('Value "%s" is neither True nor False' % val)
1057
873
1058
874
    def as_int(self, key):
1059
875
        """
1060
876
        A convenience method which coerces the specified value to an integer.
1061
877
1062
878
        If the value is an invalid literal for ``int``, a ``ValueError`` will
1063
879
        be raised.
1064
880
1065
881
        >>> a = ConfigObj()
1066
882
        >>> a['a'] = 'fish'
1067
883
        >>> a.as_int('a')
1068
884
        Traceback (most recent call last):
1069
885
        ValueError: invalid literal for int(): fish
1070
886
        >>> a['b'] = '1'
1071
887
        >>> a.as_int('b')
1072
888
        1
1073
889
        >>> a['b'] = '3.2'
1074
890
        >>> a.as_int('b')
1075
891
        Traceback (most recent call last):
1076
892
        ValueError: invalid literal for int(): 3.2
1077
893
        """
1078
894
        return int(self[key])
1079
895
1080
896
    def as_float(self, key):
1081
897
        """
1082
898
        A convenience method which coerces the specified value to a float.
1083
899
1084
900
        If the value is an invalid literal for ``float``, a ``ValueError`` will
1085
901
        be raised.
1086
902
1087
903
        >>> a = ConfigObj()
1088
904
        >>> a['a'] = 'fish'
1089
905
        >>> a.as_float('a')
1090
906
        Traceback (most recent call last):
1091
907
        ValueError: invalid literal for float(): fish
1092
908
        >>> a['b'] = '1'
1093
909
        >>> a.as_float('b')
1094
910
        1.0
1095
911
        >>> a['b'] = '3.2'
1096
912
        >>> a.as_float('b')
1097
913
        3.2000000000000002
1098
914
        """
1099
915
        return float(self[key])
1100
916
1101
917
1102
918
class ConfigObj(Section):
1103
919
    """
1104
920
    An object to read, create, and write config files.
1105
921
1106
922
    Testing with duplicate keys and sections.
1107
923
1108
924
    >>> c = '''
1109
925
    ... [hello]
1110
926
    ... member = value
1111
927
    ... [hello again]
1112
928
    ... member = value
1113
929
    ... [ "hello" ]
1114
930
    ... member = value
1115
931
    ... '''
1116
932
    >>> ConfigObj(c.split('\\n'), raise_errors = True)
1117
933
    Traceback (most recent call last):
1118
934
    DuplicateError: Duplicate section name at line 5.
1119
935
1120
936
    >>> d = '''
1121
937
    ... [hello]
1122
938
    ... member = value
1123
939
    ... [hello again]
1124
940
    ... member1 = value
1125
941
    ... member2 = value
1126
942
    ... 'member1' = value
1127
943
    ... [ "and again" ]
1128
944
    ... member = value
1129
945
    ... '''
1130
946
    >>> ConfigObj(d.split('\\n'), raise_errors = True)
1131
947
    Traceback (most recent call last):
1132
948
    DuplicateError: Duplicate keyword name at line 6.
1133
949
    """
1134
950
1135
951
    _keyword = re.compile(r'''^ # line start
1136
952
        (\s*)                   # indentation
1137
953
        (                       # keyword
1138
954
            (?:".*?")|          # double quotes
1139
955
            (?:'.*?')|          # single quotes
1140
956
            (?:[^'"=].*?)       # no quotes
1141
957
        )
1142
958
        \s*=\s*                 # divider
1143
959
        (.*)                    # value (including list values and comments)
1144
960
        $   # line end
1145
961
        ''',
1146
962
        re.VERBOSE)
1147
963
1148
964
    _sectionmarker = re.compile(r'''^
1149
965
        (\s*)                     # 1: indentation
1150
966
        ((?:\[\s*)+)              # 2: section marker open
1151
967
        (                         # 3: section name open
1152
968
            (?:"\s*\S.*?\s*")|    # at least one non-space with double quotes
1153
969
            (?:'\s*\S.*?\s*')|    # at least one non-space with single quotes
1154
970
            (?:[^'"\s].*?)        # at least one non-space unquoted
1155
971
        )                         # section name close
1156
972
        ((?:\s*\])+)              # 4: section marker close
1157
973
        \s*(\#.*)?                # 5: optional comment
1158
974
        $''',
1159
975
        re.VERBOSE)
1160
976
1161
977
    # this regexp pulls list values out as a single string
1162
978
    # or single values and comments
1163
979
    # FIXME: this regex adds a '' to the end of comma terminated lists
1164
980
    #   workaround in ``_handle_value``
1165
981
    _valueexp = re.compile(r'''^
1166
982
        (?:
1167
983
            (?:
1168
984
                (
1169
985
                    (?:
1170
986
                        (?:
1171
987
                            (?:".*?")|              # double quotes
1172
988
                            (?:'.*?')|              # single quotes
1173
989
                            (?:[^'",\#][^,\#]*?)    # unquoted
1174
990
                        )
1175
991
                        \s*,\s*                     # comma
1176
992
                    )*      # match all list items ending in a comma (if any)
1177
993
                )
1178
994
                (
1179
995
                    (?:".*?")|                      # double quotes
1180
996
                    (?:'.*?')|                      # single quotes
1181
997
                    (?:[^'",\#\s][^,]*?)|           # unquoted
1182
998
                    (?:(?<!,))                      # Empty value
1183
999
                )?          # last item in a list - or string value
1184
1000
            )|
1185
1001
            (,)             # alternatively a single comma - empty list
1186
1002
        )
1187
1003
        \s*(\#.*)?          # optional comment
1188
1004
        $''',
1189
1005
        re.VERBOSE)
1190
1006
1191
1007
    # use findall to get the members of a list value
1192
1008
    _listvalueexp = re.compile(r'''
1193
1009
        (
1194
1010
            (?:".*?")|          # double quotes
1195
1011
            (?:'.*?')|          # single quotes
1196
1012
            (?:[^'",\#].*?)       # unquoted
1197
1013
        )
1198
1014
        \s*,\s*                 # comma
1199
1015
        ''',
1200
1016
        re.VERBOSE)
1201
1017
1202
1018
    # this regexp is used for the value
1203
1019
    # when lists are switched off
1204
1020
    _nolistvalue = re.compile(r'''^
1205
1021
        (
1206
1022
            (?:".*?")|          # double quotes
1207
1023
            (?:'.*?')|          # single quotes
1208
1024
            (?:[^'"\#].*?)|     # unquoted
1209
1025
            (?:)                # Empty value
1210
1026
        )
1211
1027
        \s*(\#.*)?              # optional comment
1212
1028
        $''',
1213
1029
        re.VERBOSE)
1214
1030
1215
1031
    # regexes for finding triple quoted values on one line
1216
1032
    _single_line_single = re.compile(r"^'''(.*?)'''\s*(#.*)?$")
1217
1033
    _single_line_double = re.compile(r'^"""(.*?)"""\s*(#.*)?$')
1218
1034
    _multi_line_single = re.compile(r"^(.*?)'''\s*(#.*)?$")
1219
1035
    _multi_line_double = re.compile(r'^(.*?)"""\s*(#.*)?$')
1220
1036
1221
1037
    _triple_quote = {
1222
1038
        "'''": (_single_line_single, _multi_line_single),
1223
1039
        '"""': (_single_line_double, _multi_line_double),
1224
1040
    }
1225
1041
1226
1042
    # Used by the ``istrue`` Section method
1227
1043
    _bools = {
1228
1044
        'yes': True, 'no': False,
1229
1045
        'on': True, 'off': False,
1230
1046
        '1': True, '0': False,
1231
1047
        'true': True, 'false': False,
1232
1048
        }
1233
1049
1234
1050
    def __init__(self, infile=None, options=None, **kwargs):
1235
1051
        """
1236
1052
        Parse or create a config file object.
1237
1053
1238
1054
        ``ConfigObj(infile=None, options=None, **kwargs)``
1239
1055
        """
1240
1056
        if infile is None:
1241
1057
            infile = []
1242
1058
        if options is None:
1243
1059
            options = {}
1244
1060
        # keyword arguments take precedence over an options dictionary
1245
1061
        options.update(kwargs)
1246
1062
        # init the superclass
1247
1063
        Section.__init__(self, self, 0, self)
1248
1064
        #
1249
1065
        defaults = OPTION_DEFAULTS.copy()
1250
1066
        for entry in options.keys():
1251
1067
            if entry not in defaults.keys():
1252
1068
                raise TypeError, 'Unrecognised option "%s".' % entry
1253
1069
        # TODO: check the values too.
1254
1070
        #
1255
1071
        # Add any explicit options to the defaults
1256
1072
        defaults.update(options)
1257
1073
        #
1258
1074
        # initialise a few variables
1259
1075
        self.filename = None
1260
1076
        self._errors = []
1261
1077
        self.raise_errors = defaults['raise_errors']
1262
1078
        self.interpolation = defaults['interpolation']
1263
1079
        self.list_values = defaults['list_values']
1264
1080
        self.create_empty = defaults['create_empty']
1265
1081
        self.file_error = defaults['file_error']
1266
1082
        self.stringify = defaults['stringify']
1267
1083
        self.indent_type = defaults['indent_type']
1268
1084
        self.encoding = defaults['encoding']
1269
1085
        self.default_encoding = defaults['default_encoding']
1270
1086
        self.BOM = False
1271
1087
        self.newlines = None
1272
1088
        self.write_empty_values = defaults['write_empty_values']
1273
1089
        self.unrepr = defaults['unrepr']
1274
1090
        #
1275
1091
        self.initial_comment = []
1276
1092
        self.final_comment = []
1277
1093
        #
1278
1094
        if isinstance(infile, StringTypes):
1279
1095
            self.filename = infile
1280
1096
            if os.path.isfile(infile):
1281
1097
                infile = open(infile).read() or []
1282
1098
            elif self.file_error:
1283
1099
                # raise an error if the file doesn't exist
1284
1100
                raise IOError, 'Config file not found: "%s".' % self.filename
1285
1101
            else:
1286
1102
                # file doesn't already exist
1287
1103
                if self.create_empty:
1288
1104
                    # this is a good test that the filename specified
1289
1105
                    # isn't impossible - like on a non existent device
1290
1106
                    h = open(infile, 'w')
1291
1107
                    h.write('')
1292
1108
                    h.close()
1293
1109
                infile = []
1294
1110
        elif isinstance(infile, (list, tuple)):
1295
1111
            infile = list(infile)
1296
1112
        elif isinstance(infile, dict):
1297
1113
            # initialise self
1298
1114
            # the Section class handles creating subsections
1299
1115
            if isinstance(infile, ConfigObj):
1300
1116
                # get a copy of our ConfigObj
1301
1117
                infile = infile.dict()
1302
1118
            for entry in infile:
1303
1119
                self[entry] = infile[entry]
1304
1120
            del self._errors
1305
1121
            if defaults['configspec'] is not None:
1306
1122
                self._handle_configspec(defaults['configspec'])
1307
1123
            else:
1308
1124
                self.configspec = None
1309
1125
            return
1310
1126
        elif hasattr(infile, 'read'):
1311
1127
            # This supports file like objects
1312
1128
            infile = infile.read() or []
1313
1129
            # needs splitting into lines - but needs doing *after* decoding
1314
1130
            # in case it's not an 8 bit encoding
1315
1131
        else:
1316
1132
            raise TypeError, ('infile must be a filename,'
1317
1133
                ' file like object, or list of lines.')
1318
1134
        #
1319
1135
        if infile:
1320
1136
            # don't do it for the empty ConfigObj
1321
1137
            infile = self._handle_bom(infile)
1322
1138
            # infile is now *always* a list
1323
1139
            #
1324
1140
            # Set the newlines attribute (first line ending it finds)
1325
1141
            # and strip trailing '\n' or '\r' from lines
1326
1142
            for line in infile:
1327
1143
                if (not line) or (line[-1] not in '\r\n'):
1328
1144
                    continue
1329
1145
                for end in ('\r\n', '\n', '\r'):
1330
1146
                    if line.endswith(end):
1331
1147
                        self.newlines = end
1332
1148
                        break
1333
1149
                break
1334
1150
            infile = [line.rstrip('\r\n') for line in infile]
1335
1151
        #
1336
1152
        self._parse(infile)
1337
1153
        # if we had any errors, now is the time to raise them
1338
1154
        if self._errors:
1339
1155
            error = ConfigObjError("Parsing failed.")
1340
1156
            # set the errors attribute; it's a list of tuples:
1341
1157
            # (error_type, message, line_number)
1342
1158
            error.errors = self._errors
1343
1159
            # set the config attribute
1344
1160
            error.config = self
1345
1161
            raise error
1346
1162
        # delete private attributes
1347
1163
        del self._errors
1348
1164
        #
1349
1165
        if defaults['configspec'] is None:
1350
1166
            self.configspec = None
1351
1167
        else:
1352
1168
            self._handle_configspec(defaults['configspec'])
1353
1169
1354
1170
    def _handle_bom(self, infile):
1355
1171
        """
1356
1172
        Handle any BOM, and decode if necessary.
1357
1173
1358
1174
        If an encoding is specified, that *must* be used - but the BOM should
1359
1175
        still be removed (and the BOM attribute set).
1360
1176
1361
1177
        (If the encoding is wrongly specified, then a BOM for an alternative
1362
1178
        encoding won't be discovered or removed.)
1363
1179
1364
1180
        If an encoding is not specified, UTF8 or UTF16 BOM will be detected and
1365
1181
        removed. The BOM attribute will be set. UTF16 will be decoded to
1366
1182
        unicode.
1367
1183
1368
1184
        NOTE: This method must not be called with an empty ``infile``.
1369
1185
1370
1186
        Specifying the *wrong* encoding is likely to cause a
1371
1187
        ``UnicodeDecodeError``.
1372
1188
1373
1189
        ``infile`` must always be returned as a list of lines, but may be
1374
1190
        passed in as a single string.
1375
1191
        """
1376
1192
        if ((self.encoding is not None) and
1377
1193
            (self.encoding.lower() not in BOM_LIST)):
1378
1194
            # No need to check for a BOM
1379
1195
            # encoding specified doesn't have one
1380
1196
            # just decode
1381
1197
            return self._decode(infile, self.encoding)
1382
1198
        #
1383
1199
        if isinstance(infile, (list, tuple)):
1384
1200
            line = infile[0]
1385
1201
        else:
1386
1202
            line = infile
1387
1203
        if self.encoding is not None:
1388
1204
            # encoding explicitly supplied
1389
1205
            # And it could have an associated BOM
1390
1206
            # TODO: if encoding is just UTF16 - we ought to check for both
1391
1207
            # TODO: big endian and little endian versions.
1392
1208
            enc = BOM_LIST[self.encoding.lower()]
1393
1209
            if enc == 'utf_16':
1394
1210
                # For UTF16 we try big endian and little endian
1395
1211
                for BOM, (encoding, final_encoding) in BOMS.items():
1396
1212
                    if not final_encoding:
1397
1213
                        # skip UTF8
1398
1214
                        continue
1399
1215
                    if infile.startswith(BOM):
1400
1216
                        ### BOM discovered
1401
1217
                        ##self.BOM = True
1402
1218
                        # Don't need to remove BOM
1403
1219
                        return self._decode(infile, encoding)
1404
1220
                #
1405
1221
                # If we get this far, will *probably* raise a DecodeError
1406
1222
                # As it doesn't appear to start with a BOM
1407
1223
                return self._decode(infile, self.encoding)
1408
1224
            #
1409
1225
            # Must be UTF8
1410
1226
            BOM = BOM_SET[enc]
1411
1227
            if not line.startswith(BOM):
1412
1228
                return self._decode(infile, self.encoding)
1413
1229
            #
1414
1230
            newline = line[len(BOM):]
1415
1231
            #
1416
1232
            # BOM removed
1417
1233
            if isinstance(infile, (list, tuple)):
1418
1234
                infile[0] = newline
1419
1235
            else:
1420
1236
                infile = newline
1421
1237
            self.BOM = True
1422
1238
            return self._decode(infile, self.encoding)
1423
1239
        #
1424
1240
        # No encoding specified - so we need to check for UTF8/UTF16
1425
1241
        for BOM, (encoding, final_encoding) in BOMS.items():
1426
1242
            if not line.startswith(BOM):
1427
1243
                continue
1428
1244
            else:
1429
1245
                # BOM discovered
1430
1246
                self.encoding = final_encoding
1431
1247
                if not final_encoding:
1432
1248
                    self.BOM = True
1433
1249
                    # UTF8
1434
1250
                    # remove BOM
1435
1251
                    newline = line[len(BOM):]
1436
1252
                    if isinstance(infile, (list, tuple)):
1437
1253
                        infile[0] = newline
1438
1254
                    else:
1439
1255
                        infile = newline
1440
1256
                    # UTF8 - don't decode
1441
1257
                    if isinstance(infile, StringTypes):
1442
1258
                        return infile.splitlines(True)
1443
1259
                    else:
1444
1260
                        return infile
1445
1261
                # UTF16 - have to decode
1446
1262
                return self._decode(infile, encoding)
1447
1263
        #
1448
1264
        # No BOM discovered and no encoding specified, just return
1449
1265
        if isinstance(infile, StringTypes):
1450
1266
            # infile read from a file will be a single string
1451
1267
            return infile.splitlines(True)
1452
1268
        else:
1453
1269
            return infile
1454
1270
1455
1271
    def _a_to_u(self, string):
1456
1272
        """Decode ascii strings to unicode if a self.encoding is specified."""
1457
1273
        if not self.encoding:
1458
1274
            return string
1459
1275
        else:
1460
1276
            return string.decode('ascii')
1461
1277
1462
1278
    def _decode(self, infile, encoding):
1463
1279
        """
1464
1280
        Decode infile to unicode. Using the specified encoding.
1465
1281
1466
1282
        if is a string, it also needs converting to a list.
1467
1283
        """
1468
1284
        if isinstance(infile, StringTypes):
1469
1285
            # can't be unicode
1470
1286
            # NOTE: Could raise a ``UnicodeDecodeError``
1471
1287
            return infile.decode(encoding).splitlines(True)
1472
1288
        for i, line in enumerate(infile):
1473
1289
            if not isinstance(line, unicode):
1474
1290
                # NOTE: The isinstance test here handles mixed lists of unicode/string
1475
1291
                # NOTE: But the decode will break on any non-string values
1476
1292
                # NOTE: Or could raise a ``UnicodeDecodeError``
1477
1293
                infile[i] = line.decode(encoding)
1478
1294
        return infile
1479
1295
1480
1296
    def _decode_element(self, line):
1481
1297
        """Decode element to unicode if necessary."""
1482
1298
        if not self.encoding:
1483
1299
            return line
1484
1300
        if isinstance(line, str) and self.default_encoding:
1485
1301
            return line.decode(self.default_encoding)
1486
1302
        return line
1487
1303
1488
1304
    def _str(self, value):
1489
1305
        """
1490
1306
        Used by ``stringify`` within validate, to turn non-string values
1491
1307
        into strings.
1492
1308
        """
1493
1309
        if not isinstance(value, StringTypes):
1494
1310
            return str(value)
1495
1311
        else:
1496
1312
            return value
1497
1313
1498
1314
    def _parse(self, infile):
1499
1315
        """
1500
1316
        Actually parse the config file
1501
1317
1502
1318
        Testing Interpolation
1503
1319
1504
1320
        >>> c = ConfigObj()
1505
1321
        >>> c['DEFAULT'] = {
1506
1322
        ...     'b': 'goodbye',
1507
1323
        ...     'userdir': 'c:\\\\home',
1508
1324
        ...     'c': '%(d)s',
1509
1325
        ...     'd': '%(c)s'
1510
1326
        ... }
1511
1327
        >>> c['section'] = {
1512
1328
        ...     'a': '%(datadir)s\\\\some path\\\\file.py',
1513
1329
        ...     'b': '%(userdir)s\\\\some path\\\\file.py',
1514
1330
        ...     'c': 'Yo %(a)s',
1515
1331
        ...     'd': '%(not_here)s',
1516
1332
        ...     'e': '%(c)s',
1517
1333
        ... }
1518
1334
        >>> c['section']['DEFAULT'] = {
1519
1335
        ...     'datadir': 'c:\\\\silly_test',
1520
1336
        ...     'a': 'hello - %(b)s',
1521
1337
        ... }
1522
1338
        >>> c['section']['a'] == 'c:\\\\silly_test\\\\some path\\\\file.py'
1523
1339
        1
1524
1340
        >>> c['section']['b'] == 'c:\\\\home\\\\some path\\\\file.py'
1525
1341
        1
1526
1342
        >>> c['section']['c'] == 'Yo hello - goodbye'
1527
1343
        1
1528
1344
1529
1345
        Switching Interpolation Off
1530
1346
1531
1347
        >>> c.interpolation = False
1532
1348
        >>> c['section']['a'] == '%(datadir)s\\\\some path\\\\file.py'
1533
1349
        1
1534
1350
        >>> c['section']['b'] == '%(userdir)s\\\\some path\\\\file.py'
1535
1351
        1
1536
1352
        >>> c['section']['c'] == 'Yo %(a)s'
1537
1353
        1
1538
1354
1539
1355
        Testing the interpolation errors.
1540
1356
1541
1357
        >>> c.interpolation = True
1542
1358
        >>> c['section']['d']
1543
1359
        Traceback (most recent call last):
1544
1360
        MissingInterpolationOption: missing option "not_here" in interpolation.
1545
1361
        >>> c['section']['e']
1546
1362
        Traceback (most recent call last):
1547
1363
        InterpolationDepthError: max interpolation depth exceeded in value "%(c)s".
1548
1364
1549
1365
        Testing our quoting.
1550
1366
1551
1367
        >>> i._quote('\"""\'\'\'')
1552
1368
        Traceback (most recent call last):
1553
1369
        SyntaxError: EOF while scanning triple-quoted string
1554
1370
        >>> try:
1555
1371
        ...     i._quote('\\n', multiline=False)
1556
1372
        ... except ConfigObjError, e:
1557
1373
        ...    e.msg
1558
1374
        'Value "\\n" cannot be safely quoted.'
1559
1375
        >>> k._quote(' "\' ', multiline=False)
1560
1376
        Traceback (most recent call last):
1561
1377
        SyntaxError: EOL while scanning single-quoted string
1562
1378
1563
1379
        Testing with "stringify" off.
1564
1380
        >>> c.stringify = False
1565
1381
        >>> c['test'] = 1
1566
1382
        Traceback (most recent call last):
1567
1383
        TypeError: Value is not a string "1".
1568
1384
1569
1385
        Testing Empty values.
1570
1386
        >>> cfg_with_empty = '''
1571
1387
        ... k =
1572
1388
        ... k2 =# comment test
1573
1389
        ... val = test
1574
1390
        ... val2 = ,
1575
1391
        ... val3 = 1,
1576
1392
        ... val4 = 1, 2
1577
1393
        ... val5 = 1, 2, '''.splitlines()
1578
1394
        >>> cwe = ConfigObj(cfg_with_empty)
1579
1395
        >>> cwe == {'k': '', 'k2': '', 'val': 'test', 'val2': [],
1580
1396
        ...  'val3': ['1'], 'val4': ['1', '2'], 'val5': ['1', '2']}
1581
1397
        1
1582
1398
        >>> cwe = ConfigObj(cfg_with_empty, list_values=False)
1583
1399
        >>> cwe == {'k': '', 'k2': '', 'val': 'test', 'val2': ',',
1584
1400
        ...  'val3': '1,', 'val4': '1, 2', 'val5': '1, 2,'}
1585
1401
        1
1586
1402
        """
1587
1403
        temp_list_values = self.list_values
1588
1404
        if self.unrepr:
1589
1405
            self.list_values = False
1590
1406
        comment_list = []
1591
1407
        done_start = False
1592
1408
        this_section = self
1593
1409
        maxline = len(infile) - 1
1594
1410
        cur_index = -1
1595
1411
        reset_comment = False
1596
1412
        while cur_index < maxline:
1597
1413
            if reset_comment:
1598
1414
                comment_list = []
1599
1415
            cur_index += 1
1600
1416
            line = infile[cur_index]
1601
1417
            sline = line.strip()
1602
1418
            # do we have anything on the line ?
1603
1419
            if not sline or sline.startswith('#'):
1604
1420
                reset_comment = False
1605
1421
                comment_list.append(line)
1606
1422
                continue
1607
1423
            if not done_start:
1608
1424
                # preserve initial comment
1609
1425
                self.initial_comment = comment_list
1610
1426
                comment_list = []
1611
1427
                done_start = True
1612
1428
            reset_comment = True
1613
1429
            # first we check if it's a section marker
1614
1430
            mat = self._sectionmarker.match(line)
1615
1431
##            print >> sys.stderr, sline, mat
1616
1432
            if mat is not None:
1617
1433
                # is a section line
1618
1434
                (indent, sect_open, sect_name, sect_close, comment) = (
1619
1435
                    mat.groups())
1620
1436
                if indent and (self.indent_type is None):
1621
1437
                    self.indent_type = indent[0]
1622
1438
                cur_depth = sect_open.count('[')
1623
1439
                if cur_depth != sect_close.count(']'):
1624
1440
                    self._handle_error(
1625
1441
                        "Cannot compute the section depth at line %s.",
1626
1442
                        NestingError, infile, cur_index)
1627
1443
                    continue
1628
1444
                if cur_depth < this_section.depth:
1629
1445
                    # the new section is dropping back to a previous level
1630
1446
                    try:
1631
1447
                        parent = self._match_depth(
1632
1448
                            this_section,
1633
1449
                            cur_depth).parent
1634
1450
                    except SyntaxError:
1635
1451
                        self._handle_error(
1636
1452
                            "Cannot compute nesting level at line %s.",
1637
1453
                            NestingError, infile, cur_index)
1638
1454
                        continue
1639
1455
                elif cur_depth == this_section.depth:
1640
1456
                    # the new section is a sibling of the current section
1641
1457
                    parent = this_section.parent
1642
1458
                elif cur_depth == this_section.depth + 1:
1643
1459
                    # the new section is a child the current section
1644
1460
                    parent = this_section
1645
1461
                else:
1646
1462
                    self._handle_error(
1647
1463
                        "Section too nested at line %s.",
1648
1464
                        NestingError, infile, cur_index)
1649
1465
                #
1650
1466
                sect_name = self._unquote(sect_name)
1651
1467
                if parent.has_key(sect_name):
1652
1468
##                    print >> sys.stderr, sect_name
1653
1469
                    self._handle_error(
1654
1470
                        'Duplicate section name at line %s.',
1655
1471
                        DuplicateError, infile, cur_index)
1656
1472
                    continue
1657
1473
                # create the new section
1658
1474
                this_section = Section(
1659
1475
                    parent,
1660
1476
                    cur_depth,
1661
1477
                    self,
1662
1478
                    name=sect_name)
1663
1479
                parent[sect_name] = this_section
1664
1480
                parent.inline_comments[sect_name] = comment
1665
1481
                parent.comments[sect_name] = comment_list
1666
1482
##                print >> sys.stderr, parent[sect_name] is this_section
1667
1483
                continue
1668
1484
            #
1669
1485
            # it's not a section marker,
1670
1486
            # so it should be a valid ``key = value`` line
1671
1487
            mat = self._keyword.match(line)
1672
1488
##            print >> sys.stderr, sline, mat
1673
1489
            if mat is not None:
1674
1490
                # is a keyword value
1675
1491
                # value will include any inline comment
1676
1492
                (indent, key, value) = mat.groups()
1677
1493
                if indent and (self.indent_type is None):
1678
1494
                    self.indent_type = indent[0]
1679
1495
                # check for a multiline value
1680
1496
                if value[:3] in ['"""', "'''"]:
1681
1497
                    try:
1682
1498
                        (value, comment, cur_index) = self._multiline(
1683
1499
                            value, infile, cur_index, maxline)
1684
1500
                    except SyntaxError:
1685
1501
                        self._handle_error(
1686
1502
                            'Parse error in value at line %s.',
1687
1503
                            ParseError, infile, cur_index)
1688
1504
                        continue
1689
1505
                    else:
1690
1506
                        if self.unrepr:
1691
1507
                            value = unrepr(value)
1692
1508
                else:
1693
1509
                    # extract comment and lists
1694
1510
                    try:
1695
1511
                        (value, comment) = self._handle_value(value)
1696
1512
                    except SyntaxError:
1697
1513
                        self._handle_error(
1698
1514
                            'Parse error in value at line %s.',
1699
1515
                            ParseError, infile, cur_index)
1700
1516
                        continue
1701
1517
                #
1702
1518
##                print >> sys.stderr, sline
1703
1519
                key = self._unquote(key)
1704
1520
                if this_section.has_key(key):
1705
1521
                    self._handle_error(
1706
1522
                        'Duplicate keyword name at line %s.',
1707
1523
                        DuplicateError, infile, cur_index)
1708
1524
                    continue
1709
1525
                # add the key
1710
1526
##                print >> sys.stderr, this_section.name
1711
1527
                # we set unrepr because if we have got this far we will never
1712
1528
                # be creating a new section
1713
1529
                this_section.__setitem__(key, value, unrepr=True)
1714
1530
                this_section.inline_comments[key] = comment
1715
1531
                this_section.comments[key] = comment_list
1716
1532
##                print >> sys.stderr, key, this_section[key]
1717
1533
##                if this_section.name is not None:
1718
1534
##                    print >> sys.stderr, this_section
1719
1535
##                    print >> sys.stderr, this_section.parent
1720
1536
##                    print >> sys.stderr, this_section.parent[this_section.name]
1721
1537
                continue
1722
1538
            #
1723
1539
            # it neither matched as a keyword
1724
1540
            # or a section marker
1725
1541
            self._handle_error(
1726
1542
                'Invalid line at line "%s".',
1727
1543
                ParseError, infile, cur_index)
1728
1544
        if self.indent_type is None:
1729
1545
            # no indentation used, set the type accordingly
1730
1546
            self.indent_type = ''
1731
1547
        # preserve the final comment
1732
1548
        if not self and not self.initial_comment:
1733
1549
            self.initial_comment = comment_list
1734
1550
        elif not reset_comment:
1735
1551
            self.final_comment = comment_list
1736
1552
        self.list_values = temp_list_values
1737
1553
1738
1554
    def _match_depth(self, sect, depth):
1739
1555
        """
1740
1556
        Given a section and a depth level, walk back through the sections
1741
1557
        parents to see if the depth level matches a previous section.
1742
1558
1743
1559
        Return a reference to the right section,
1744
1560
        or raise a SyntaxError.
1745
1561
        """
1746
1562
        while depth < sect.depth:
1747
1563
            if sect is sect.parent:
1748
1564
                # we've reached the top level already
1749
1565
                raise SyntaxError
1750
1566
            sect = sect.parent
1751
1567
        if sect.depth == depth:
1752
1568
            return sect
1753
1569
        # shouldn't get here
1754
1570
        raise SyntaxError
1755
1571
1756
1572
    def _handle_error(self, text, ErrorClass, infile, cur_index):
1757
1573
        """
1758
1574
        Handle an error according to the error settings.
1759
1575
1760
1576
        Either raise the error or store it.
1761
1577
        The error will have occured at ``cur_index``
1762
1578
        """
1763
1579
        line = infile[cur_index]
1764
1580
        message = text % cur_index
1765
1581
        error = ErrorClass(message, cur_index, line)
1766
1582
        if self.raise_errors:
1767
1583
            # raise the error - parsing stops here
1768
1584
            raise error
1769
1585
        # store the error
1770
1586
        # reraise when parsing has finished
1771
1587
        self._errors.append(error)
1772
1588
1773
1589
    def _unquote(self, value):
1774
1590
        """Return an unquoted version of a value"""
1775
1591
        if (value[0] == value[-1]) and (value[0] in ('"', "'")):
1776
1592
            value = value[1:-1]
1777
1593
        return value
1778
1594
1779
1595
    def _quote(self, value, multiline=True):
1780
1596
        """
1781
1597
        Return a safely quoted version of a value.
1782
1598
1783
1599
        Raise a ConfigObjError if the value cannot be safely quoted.
1784
1600
        If multiline is ``True`` (default) then use triple quotes
1785
1601
        if necessary.
1786
1602
1787
1603
        Don't quote values that don't need it.
1788
1604
        Recursively quote members of a list and return a comma joined list.
1789
1605
        Multiline is ``False`` for lists.
1790
1606
        Obey list syntax for empty and single member lists.
1791
1607
1792
1608
        If ``list_values=False`` then the value is only quoted if it contains
1793
1609
        a ``\n`` (is multiline).
1794
1610
1795
1611
        If ``write_empty_values`` is set, and the value is an empty string, it
1796
1612
        won't be quoted.
1797
1613
        """
1798
1614
        if multiline and self.write_empty_values and value == '':
1799
1615
            # Only if multiline is set, so that it is used for values not
1800
1616
            # keys, and not values that are part of a list
1801
1617
            return ''
1802
1618
        if multiline and isinstance(value, (list, tuple)):
1803
1619
            if not value:
1804
1620
                return ','
1805
1621
            elif len(value) == 1:
1806
1622
                return self._quote(value[0], multiline=False) + ','
1807
1623
            return ', '.join([self._quote(val, multiline=False)
1808
1624
                for val in value])
1809
1625
        if not isinstance(value, StringTypes):
1810
1626
            if self.stringify:
1811
1627
                value = str(value)
1812
1628
            else:
1813
1629
                raise TypeError, 'Value "%s" is not a string.' % value
1814
1630
        squot = "'%s'"
1815
1631
        dquot = '"%s"'
1816
1632
        noquot = "%s"
1817
1633
        wspace_plus = ' \r\t\n\v\t\'"'
1818
1634
        tsquot = '"""%s"""'
1819
1635
        tdquot = "'''%s'''"
1820
1636
        if not value:
1821
1637
            return '""'
1822
1638
        if (not self.list_values and '\n' not in value) or not (multiline and
1823
1639
                ((("'" in value) and ('"' in value)) or ('\n' in value))):
1824
1640
            if not self.list_values:
1825
1641
                # we don't quote if ``list_values=False``
1826
1642
                quot = noquot
1827
1643
            # for normal values either single or double quotes will do
1828
1644
            elif '\n' in value:
1829
1645
                # will only happen if multiline is off - e.g. '\n' in key
1830
1646
                raise ConfigObjError, ('Value "%s" cannot be safely quoted.' %
1831
1647
                    value)
1832
1648
            elif ((value[0] not in wspace_plus) and
1833
1649
                    (value[-1] not in wspace_plus) and
1834
1650
                    (',' not in value)):
1835
1651
                quot = noquot
1836
1652
            else:
1837
1653
                if ("'" in value) and ('"' in value):
1838
1654
                    raise ConfigObjError, (
1839
1655
                        'Value "%s" cannot be safely quoted.' % value)
1840
1656
                elif '"' in value:
1841
1657
                    quot = squot
1842
1658
                else:
1843
1659
                    quot = dquot
1844
1660
        else:
1845
1661
            # if value has '\n' or "'" *and* '"', it will need triple quotes
1846
1662
            if (value.find('"""') != -1) and (value.find("'''") != -1):
1847
1663
                raise ConfigObjError, (
1848
1664
                    'Value "%s" cannot be safely quoted.' % value)
1849
1665
            if value.find('"""') == -1:
1850
1666
                quot = tdquot
1851
1667
            else:
1852
1668
                quot = tsquot
1853
1669
        return quot % value
1854
1670
1855
1671
    def _handle_value(self, value):
1856
1672
        """
1857
1673
        Given a value string, unquote, remove comment,
1858
1674
        handle lists. (including empty and single member lists)
1859
1675
1860
1676
        Testing list values.
1861
1677
1862
1678
        >>> testconfig3 = '''
1863
1679
        ... a = ,
1864
1680
        ... b = test,
1865
1681
        ... c = test1, test2   , test3
1866
1682
        ... d = test1, test2, test3,
1867
1683
        ... '''
1868
1684
        >>> d = ConfigObj(testconfig3.split('\\n'), raise_errors=True)
1869
1685
        >>> d['a'] == []
1870
1686
        1
1871
1687
        >>> d['b'] == ['test']
1872
1688
        1
1873
1689
        >>> d['c'] == ['test1', 'test2', 'test3']
1874
1690
        1
1875
1691
        >>> d['d'] == ['test1', 'test2', 'test3']
1876
1692
        1
1877
1693
1878
1694
        Testing with list values off.
1879
1695
1880
1696
        >>> e = ConfigObj(
1881
1697
        ...     testconfig3.split('\\n'),
1882
1698
        ...     raise_errors=True,
1883
1699
        ...     list_values=False)
1884
1700
        >>> e['a'] == ','
1885
1701
        1
1886
1702
        >>> e['b'] == 'test,'
1887
1703
        1
1888
1704
        >>> e['c'] == 'test1, test2   , test3'
1889
1705
        1
1890
1706
        >>> e['d'] == 'test1, test2, test3,'
1891
1707
        1
1892
1708
1893
1709
        Testing creating from a dictionary.
1894
1710
1895
1711
        >>> f = {
1896
1712
        ...     'key1': 'val1',
1897
1713
        ...     'key2': 'val2',
1898
1714
        ...     'section 1': {
1899
1715
        ...         'key1': 'val1',
1900
1716
        ...         'key2': 'val2',
1901
1717
        ...         'section 1b': {
1902
1718
        ...             'key1': 'val1',
1903
1719
        ...             'key2': 'val2',
1904
1720
        ...         },
1905
1721
        ...     },
1906
1722
        ...     'section 2': {
1907
1723
        ...         'key1': 'val1',
1908
1724
        ...         'key2': 'val2',
1909
1725
        ...         'section 2b': {
1910
1726
        ...             'key1': 'val1',
1911
1727
        ...             'key2': 'val2',
1912
1728
        ...         },
1913
1729
        ...     },
1914
1730
        ...      'key3': 'val3',
1915
1731
        ... }
1916
1732
        >>> g = ConfigObj(f)
1917
1733
        >>> f == g
1918
1734
        1
1919
1735
1920
1736
        Testing we correctly detect badly built list values (4 of them).
1921
1737
1922
1738
        >>> testconfig4 = '''
1923
1739
        ... config = 3,4,,
1924
1740
        ... test = 3,,4
1925
1741
        ... fish = ,,
1926
1742
        ... dummy = ,,hello, goodbye
1927
1743
        ... '''
1928
1744
        >>> try:
1929
1745
        ...     ConfigObj(testconfig4.split('\\n'))
1930
1746
        ... except ConfigObjError, e:
1931
1747
        ...     len(e.errors)
1932
1748
        4
1933
1749
1934
1750
        Testing we correctly detect badly quoted values (4 of them).
1935
1751
1936
1752
        >>> testconfig5 = '''
1937
1753
        ... config = "hello   # comment
1938
1754
        ... test = 'goodbye
1939
1755
        ... fish = 'goodbye   # comment
1940
1756
        ... dummy = "hello again
1941
1757
        ... '''
1942
1758
        >>> try:
1943
1759
        ...     ConfigObj(testconfig5.split('\\n'))
1944
1760
        ... except ConfigObjError, e:
1945
1761
        ...     len(e.errors)
1946
1762
        4
1947
1763
        """
1948
1764
        # do we look for lists in values ?
1949
1765
        if not self.list_values:
1950
1766
            mat = self._nolistvalue.match(value)
1951
1767
            if mat is None:
1952
1768
                raise SyntaxError
1953
1769
            (value, comment) = mat.groups()
1954
1770
            if not self.unrepr:
1955
1771
                # NOTE: we don't unquote here
1956
1772
                return (value, comment)
1957
1773
            else:
1958
1774
                return (unrepr(value), comment)
1959
1775
        mat = self._valueexp.match(value)
1960
1776
        if mat is None:
1961
1777
            # the value is badly constructed, probably badly quoted,
1962
1778
            # or an invalid list
1963
1779
            raise SyntaxError
1964
1780
        (list_values, single, empty_list, comment) = mat.groups()
1965
1781
        if (list_values == '') and (single is None):
1966
1782
            # change this if you want to accept empty values
1967
1783
            raise SyntaxError
1968
1784
        # NOTE: note there is no error handling from here if the regex
1969
1785
        # is wrong: then incorrect values will slip through
1970
1786
        if empty_list is not None:
1971
1787
            # the single comma - meaning an empty list
1972
1788
            return ([], comment)
1973
1789
        if single is not None:
1974
1790
            # handle empty values
1975
1791
            if list_values and not single:
1976
1792
                # FIXME: the '' is a workaround because our regex now matches
1977
1793
                #   '' at the end of a list if it has a trailing comma
1978
1794
                single = None
1979
1795
            else:
1980
1796
                single = single or '""'
1981
1797
                single = self._unquote(single)
1982
1798
        if list_values == '':
1983
1799
            # not a list value
1984
1800
            return (single, comment)
1985
1801
        the_list = self._listvalueexp.findall(list_values)
1986
1802
        the_list = [self._unquote(val) for val in the_list]
1987
1803
        if single is not None:
1988
1804
            the_list += [single]
1989
1805
        return (the_list, comment)
1990
1806
1991
1807
    def _multiline(self, value, infile, cur_index, maxline):
1992
1808
        """
1993
1809
        Extract the value, where we are in a multiline situation
1994
1810
1995
1811
        Testing multiline values.
1996
1812
1997
1813
        >>> i == {
1998
1814
        ...     'name4': ' another single line value ',
1999
1815
        ...     'multi section': {
2000
1816
        ...         'name4': '\\n        Well, this is a\\n        multiline '
2001
1817
        ...             'value\\n        ',
2002
1818
        ...         'name2': '\\n        Well, this is a\\n        multiline '
2003
1819
        ...             'value\\n        ',
2004
1820
        ...         'name3': '\\n        Well, this is a\\n        multiline '
2005
1821
        ...             'value\\n        ',
2006
1822
        ...         'name1': '\\n        Well, this is a\\n        multiline '
2007
1823
        ...             'value\\n        ',
2008
1824
        ...     },
2009
1825
        ...     'name2': ' another single line value ',
2010
1826
        ...     'name3': ' a single line value ',
2011
1827
        ...     'name1': ' a single line value ',
2012
1828
        ... }
2013
1829
        1
2014
1830
        """
2015
1831
        quot = value[:3]
2016
1832
        newvalue = value[3:]
2017
1833
        single_line = self._triple_quote[quot][0]
2018
1834
        multi_line = self._triple_quote[quot][1]
2019
1835
        mat = single_line.match(value)
2020
1836
        if mat is not None:
2021
1837
            retval = list(mat.groups())
2022
1838
            retval.append(cur_index)
2023
1839
            return retval
2024
1840
        elif newvalue.find(quot) != -1:
2025
1841
            # somehow the triple quote is missing
2026
1842
            raise SyntaxError
2027
1843
        #
2028
1844
        while cur_index < maxline:
2029
1845
            cur_index += 1
2030
1846
            newvalue += '\n'
2031
1847
            line = infile[cur_index]
2032
1848
            if line.find(quot) == -1:
2033
1849
                newvalue += line
2034
1850
            else:
2035
1851
                # end of multiline, process it
2036
1852
                break
2037
1853
        else:
2038
1854
            # we've got to the end of the config, oops...
2039
1855
            raise SyntaxError
2040
1856
        mat = multi_line.match(line)
2041
1857
        if mat is None:
2042
1858
            # a badly formed line
2043
1859
            raise SyntaxError
2044
1860
        (value, comment) = mat.groups()
2045
1861
        return (newvalue + value, comment, cur_index)
2046
1862
2047
1863
    def _handle_configspec(self, configspec):
2048
1864
        """Parse the configspec."""
2049
1865
        # FIXME: Should we check that the configspec was created with the
2050
1866
        #   correct settings ? (i.e. ``list_values=False``)
2051
1867
        if not isinstance(configspec, ConfigObj):
2052
1868
            try:
2053
1869
                configspec = ConfigObj(
2054
1870
                    configspec,
2055
1871
                    raise_errors=True,
2056
1872
                    file_error=True,
2057
1873
                    list_values=False)
2058
1874
            except ConfigObjError, e:
2059
1875
                # FIXME: Should these errors have a reference
2060
1876
                # to the already parsed ConfigObj ?
2061
1877
                raise ConfigspecError('Parsing configspec failed: %s' % e)
2062
1878
            except IOError, e:
2063
1879
                raise IOError('Reading configspec failed: %s' % e)
2064
1880
        self._set_configspec_value(configspec, self)
2065
1881
2066
1882
    def _set_configspec_value(self, configspec, section):
2067
1883
        """Used to recursively set configspec values."""
2068
1884
        if '__many__' in configspec.sections:
2069
1885
            section.configspec['__many__'] = configspec['__many__']
2070
1886
            if len(configspec.sections) > 1:
2071
1887
                # FIXME: can we supply any useful information here ?
2072
1888
                raise RepeatSectionError
2073
1889
        if hasattr(configspec, 'initial_comment'):
2074
1890
            section._configspec_initial_comment = configspec.initial_comment
2075
1891
            section._configspec_final_comment = configspec.final_comment
2076
1892
            section._configspec_encoding = configspec.encoding
2077
1893
            section._configspec_BOM = configspec.BOM
2078
1894
            section._configspec_newlines = configspec.newlines
2079
1895
            section._configspec_indent_type = configspec.indent_type
2080
1896
        for entry in configspec.scalars:
2081
1897
            section._configspec_comments[entry] = configspec.comments[entry]
2082
1898
            section._configspec_inline_comments[entry] = (
2083
1899
                configspec.inline_comments[entry])
2084
1900
            section.configspec[entry] = configspec[entry]
2085
1901
            section._order.append(entry)
2086
1902
        for entry in configspec.sections:
2087
1903
            if entry == '__many__':
2088
1904
                continue
2089
1905
            section._cs_section_comments[entry] = configspec.comments[entry]
2090
1906
            section._cs_section_inline_comments[entry] = (
2091
1907
                configspec.inline_comments[entry])
2092
1908
            if not section.has_key(entry):
2093
1909
                section[entry] = {}
2094
1910
            self._set_configspec_value(configspec[entry], section[entry])
2095
1911
2096
1912
    def _handle_repeat(self, section, configspec):
2097
1913
        """Dynamically assign configspec for repeated section."""
2098
1914
        try:
2099
1915
            section_keys = configspec.sections
2100
1916
            scalar_keys = configspec.scalars
2101
1917
        except AttributeError:
2102
1918
            section_keys = [entry for entry in configspec
2103
1919
                                if isinstance(configspec[entry], dict)]
2104
1920
            scalar_keys = [entry for entry in configspec
2105
1921
                                if not isinstance(configspec[entry], dict)]
2106
1922
        if '__many__' in section_keys and len(section_keys) > 1:
2107
1923
            # FIXME: can we supply any useful information here ?
2108
1924
            raise RepeatSectionError
2109
1925
        scalars = {}
2110
1926
        sections = {}
2111
1927
        for entry in scalar_keys:
2112
1928
            val = configspec[entry]
2113
1929
            scalars[entry] = val
2114
1930
        for entry in section_keys:
2115
1931
            val = configspec[entry]
2116
1932
            if entry == '__many__':
2117
1933
                scalars[entry] = val
2118
1934
                continue
2119
1935
            sections[entry] = val
2120
1936
        #
2121
1937
        section.configspec = scalars
2122
1938
        for entry in sections:
2123
1939
            if not section.has_key(entry):
2124
1940
                section[entry] = {}
2125
1941
            self._handle_repeat(section[entry], sections[entry])
2126
1942
2127
1943
    def _write_line(self, indent_string, entry, this_entry, comment):
2128
1944
        """Write an individual line, for the write method"""
2129
1945
        # NOTE: the calls to self._quote here handles non-StringType values.
2130
1946
        if not self.unrepr:
2131
1947
            val = self._decode_element(self._quote(this_entry))
2132
1948
        else:
2133
1949
            val = repr(this_entry)
2134
1950
        return '%s%s%s%s%s' % (
2135
1951
            indent_string,
2136
1952
            self._decode_element(self._quote(entry, multiline=False)),
2137
1953
            self._a_to_u(' = '),
2138
1954
            val,
2139
1955
            self._decode_element(comment))
2140
1956
2141
1957
    def _write_marker(self, indent_string, depth, entry, comment):
2142
1958
        """Write a section marker line"""
2143
1959
        return '%s%s%s%s%s' % (
2144
1960
            indent_string,
2145
1961
            self._a_to_u('[' * depth),
2146
1962
            self._quote(self._decode_element(entry), multiline=False),
2147
1963
            self._a_to_u(']' * depth),
2148
1964
            self._decode_element(comment))
2149
1965
2150
1966
    def _handle_comment(self, comment):
2151
1967
        """
2152
1968
        Deal with a comment.
2153
1969
2154
1970
        >>> filename = a.filename
2155
1971
        >>> a.filename = None
2156
1972
        >>> values = a.write()
2157
1973
        >>> index = 0
2158
1974
        >>> while index < 23:
2159
1975
        ...     index += 1
2160
1976
        ...     line = values[index-1]
2161
1977
        ...     assert line.endswith('# comment ' + str(index))
2162
1978
        >>> a.filename = filename
2163
1979
2164
1980
        >>> start_comment = ['# Initial Comment', '', '#']
2165
1981
        >>> end_comment = ['', '#', '# Final Comment']
2166
1982
        >>> newconfig = start_comment + testconfig1.split('\\n') + end_comment
2167
1983
        >>> nc = ConfigObj(newconfig)
2168
1984
        >>> nc.initial_comment
2169
1985
        ['# Initial Comment', '', '#']
2170
1986
        >>> nc.final_comment
2171
1987
        ['', '#', '# Final Comment']
2172
1988
        >>> nc.initial_comment == start_comment
2173
1989
        1
2174
1990
        >>> nc.final_comment == end_comment
2175
1991
        1
2176
1992
        """
2177
1993
        if not comment:
2178
1994
            return ''
2179
1995
        if self.indent_type == '\t':
2180
1996
            start = self._a_to_u('\t')
2181
1997
        else:
2182
1998
            start = self._a_to_u(' ' * NUM_INDENT_SPACES)
2183
1999
        if not comment.startswith('#'):
2184
2000
            start += self._a_to_u('# ')
2185
2001
        return (start + comment)
2186
2002
2187
2003
    def _compute_indent_string(self, depth):
2188
2004
        """
2189
2005
        Compute the indent string, according to current indent_type and depth
2190
2006
        """
2191
2007
        if self.indent_type == '':
2192
2008
            # no indentation at all
2193
2009
            return ''
2194
2010
        if self.indent_type == '\t':
2195
2011
            return '\t' * depth
2196
2012
        if self.indent_type == ' ':
2197
2013
            return ' ' * NUM_INDENT_SPACES * depth
2198
2014
        raise SyntaxError
2199
2015
2200
2016
    # Public methods
2201
2017
2202
2018
    def write(self, outfile=None, section=None):
2203
2019
        """
2204
2020
        Write the current ConfigObj as a file
2205
2021
2206
2022
        tekNico: FIXME: use StringIO instead of real files
2207
2023
2208
2024
        >>> filename = a.filename
2209
2025
        >>> a.filename = 'test.ini'
2210
2026
        >>> a.write()
2211
2027
        >>> a.filename = filename
2212
2028
        >>> a == ConfigObj('test.ini', raise_errors=True)
2213
2029
        1
2214
2030
        >>> os.remove('test.ini')
2215
2031
        >>> b.filename = 'test.ini'
2216
2032
        >>> b.write()
2217
2033
        >>> b == ConfigObj('test.ini', raise_errors=True)
2218
2034
        1
2219
2035
        >>> os.remove('test.ini')
2220
2036
        >>> i.filename = 'test.ini'
2221
2037
        >>> i.write()
2222
2038
        >>> i == ConfigObj('test.ini', raise_errors=True)
2223
2039
        1
2224
2040
        >>> os.remove('test.ini')
2225
2041
        >>> a = ConfigObj()
2226
2042
        >>> a['DEFAULT'] = {'a' : 'fish'}
2227
2043
        >>> a['a'] = '%(a)s'
2228
2044
        >>> a.write()
2229
2045
        ['a = %(a)s', '[DEFAULT]', 'a = fish']
2230
2046
        """
2231
2047
        if self.indent_type is None:
2232
2048
            # this can be true if initialised from a dictionary
2233
2049
            self.indent_type = DEFAULT_INDENT_TYPE
2234
2050
        #
2235
2051
        out = []
2236
2052
        cs = self._a_to_u('#')
2237
2053
        csp = self._a_to_u('# ')
2238
2054
        if section is None:
2239
2055
            int_val = self.interpolation
2240
2056
            self.interpolation = False
2241
2057
            section = self
2242
2058
            for line in self.initial_comment:
2243
2059
                line = self._decode_element(line)
2244
2060
                stripped_line = line.strip()
2245
2061
                if stripped_line and not stripped_line.startswith(cs):
2246
2062
                    line = csp + line
2247
2063
                out.append(line)
2248
2064
        #
2249
2065
        indent_string = self._a_to_u(
2250
2066
            self._compute_indent_string(section.depth))
2251
2067
        for entry in (section.scalars + section.sections):
2252
2068
            if entry in section.defaults:
2253
2069
                # don't write out default values
2254
2070
                continue
2255
2071
            for comment_line in section.comments[entry]:
2256
2072
                comment_line = self._decode_element(comment_line.lstrip())
2257
2073
                if comment_line and not comment_line.startswith(cs):
2258
2074
                    comment_line = csp + comment_line
2259
2075
                out.append(indent_string + comment_line)
2260
2076
            this_entry = section[entry]
2261
2077
            comment = self._handle_comment(section.inline_comments[entry])
2262
2078
            #
2263
2079
            if isinstance(this_entry, dict):
2264
2080
                # a section
2265
2081
                out.append(self._write_marker(
2266
2082
                    indent_string,
2267
2083
                    this_entry.depth,
2268
2084
                    entry,
2269
2085
                    comment))
2270
2086
                out.extend(self.write(section=this_entry))
2271
2087
            else:
2272
2088
                out.append(self._write_line(
2273
2089
                    indent_string,
2274
2090
                    entry,
2275
2091
                    this_entry,
2276
2092
                    comment))
2277
2093
        #
2278
2094
        if section is self:
2279
2095
            for line in self.final_comment:
2280
2096
                line = self._decode_element(line)
2281
2097
                stripped_line = line.strip()
2282
2098
                if stripped_line and not stripped_line.startswith(cs):
2283
2099
                    line = csp + line
2284
2100
                out.append(line)
2285
2101
            self.interpolation = int_val
2286
2102
        #
2287
2103
        if section is not self:
2288
2104
            return out
2289
2105
        #
2290
2106
        if (self.filename is None) and (outfile is None):
2291
2107
            # output a list of lines
2292
2108
            # might need to encode
2293
2109
            # NOTE: This will *screw* UTF16, each line will start with the BOM
2294
2110
            if self.encoding:
2295
2111
                out = [l.encode(self.encoding) for l in out]
2296
2112
            if (self.BOM and ((self.encoding is None) or
2297
2113
                (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
2298
2114
                # Add the UTF8 BOM
2299
2115
                if not out:
2300
2116
                    out.append('')
2301
2117
                out[0] = BOM_UTF8 + out[0]
2302
2118
            return out
2303
2119
        #
2304
2120
        # Turn the list to a string, joined with correct newlines
2305
2121
        output = (self._a_to_u(self.newlines or os.linesep)).join(out)
2306
2122
        if self.encoding:
2307
2123
            output = output.encode(self.encoding)
2308
2124
        if (self.BOM and ((self.encoding is None) or
2309
2125
            (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
2310
2126
            # Add the UTF8 BOM
2311
2127
            output = BOM_UTF8 + output
2312
2128
        if outfile is not None:
2313
2129
            outfile.write(output)
2314
2130
        else:
2315
2131
            h = open(self.filename, 'wb')
2316
2132
            h.write(output)
2317
2133
            h.close()
2318
2134
2319
2135
    def validate(self, validator, preserve_errors=False, copy=False,
2320
2136
        section=None):
2321
2137
        """
2322
2138
        Test the ConfigObj against a configspec.
2323
2139
2324
2140
        It uses the ``validator`` object from *validate.py*.
2325
2141
2326
2142
        To run ``validate`` on the current ConfigObj, call: ::
2327
2143
2328
2144
            test = config.validate(validator)
2329
2145
2330
2146
        (Normally having previously passed in the configspec when the ConfigObj
2331
2147
        was created - you can dynamically assign a dictionary of checks to the
2332
2148
        ``configspec`` attribute of a section though).
2333
2149
2334
2150
        It returns ``True`` if everything passes, or a dictionary of
2335
2151
        pass/fails (True/False). If every member of a subsection passes, it
2336
2152
        will just have the value ``True``. (It also returns ``False`` if all
2337
2153
        members fail).
2338
2154
2339
2155
        In addition, it converts the values from strings to their native
2340
2156
        types if their checks pass (and ``stringify`` is set).
2341
2157
2342
2158
        If ``preserve_errors`` is ``True`` (``False`` is default) then instead
2343
2159
        of a marking a fail with a ``False``, it will preserve the actual
2344
2160
        exception object. This can contain info about the reason for failure.
2345
2161
        For example the ``VdtValueTooSmallError`` indeicates that the value
2346
2162
        supplied was too small. If a value (or section) is missing it will
2347
2163
        still be marked as ``False``.
2348
2164
2349
2165
        You must have the validate module to use ``preserve_errors=True``.
2350
2166
2351
2167
        You can then use the ``flatten_errors`` function to turn your nested
2352
2168
        results dictionary into a flattened list of failures - useful for
2353
2169
        displaying meaningful error messages.
2354
2170
2355
2171
        >>> try:
2356
2172
        ...     from validate import Validator
2357
2173
        ... except ImportError:
2358
2174
        ...     print >> sys.stderr, 'Cannot import the Validator object, skipping the related tests'
2359
2175
        ... else:
2360
2176
        ...     config = '''
2361
2177
        ...     test1=40
2362
2178
        ...     test2=hello
2363
2179
        ...     test3=3
2364
2180
        ...     test4=5.0
2365
2181
        ...     [section]
2366
2182
        ...         test1=40
2367
2183
        ...         test2=hello
2368
2184
        ...         test3=3
2369
2185
        ...         test4=5.0
2370
2186
        ...         [[sub section]]
2371
2187
        ...             test1=40
2372
2188
        ...             test2=hello
2373
2189
        ...             test3=3
2374
2190
        ...             test4=5.0
2375
2191
        ... '''.split('\\n')
2376
2192
        ...     configspec = '''
2377
2193
        ...     test1= integer(30,50)
2378
2194
        ...     test2= string
2379
2195
        ...     test3=integer
2380
2196
        ...     test4=float(6.0)
2381
2197
        ...     [section ]
2382
2198
        ...         test1=integer(30,50)
2383
2199
        ...         test2=string
2384
2200
        ...         test3=integer
2385
2201
        ...         test4=float(6.0)
2386
2202
        ...         [[sub section]]
2387
2203
        ...             test1=integer(30,50)
2388
2204
        ...             test2=string
2389
2205
        ...             test3=integer
2390
2206
        ...             test4=float(6.0)
2391
2207
        ...     '''.split('\\n')
2392
2208
        ...     val = Validator()
2393
2209
        ...     c1 = ConfigObj(config, configspec=configspec)
2394
2210
        ...     test = c1.validate(val)
2395
2211
        ...     test == {
2396
2212
        ...         'test1': True,
2397
2213
        ...         'test2': True,
2398
2214
        ...         'test3': True,
2399
2215
        ...         'test4': False,
2400
2216
        ...         'section': {
2401
2217
        ...             'test1': True,
2402
2218
        ...             'test2': True,
2403
2219
        ...             'test3': True,
2404
2220
        ...             'test4': False,
2405
2221
        ...             'sub section': {
2406
2222
        ...                 'test1': True,
2407
2223
        ...                 'test2': True,
2408
2224
        ...                 'test3': True,
2409
2225
        ...                 'test4': False,
2410
2226
        ...             },
2411
2227
        ...         },
2412
2228
        ...     }
2413
2229
        1
2414
2230
        >>> val.check(c1.configspec['test4'], c1['test4'])
2415
2231
        Traceback (most recent call last):
2416
2232
        VdtValueTooSmallError: the value "5.0" is too small.
2417
2233
2418
2234
        >>> val_test_config = '''
2419
2235
        ...     key = 0
2420
2236
        ...     key2 = 1.1
2421
2237
        ...     [section]
2422
2238
        ...     key = some text
2423
2239
        ...     key2 = 1.1, 3.0, 17, 6.8
2424
2240
        ...         [[sub-section]]
2425
2241
        ...         key = option1
2426
2242
        ...         key2 = True'''.split('\\n')
2427
2243
        >>> val_test_configspec = '''
2428
2244
        ...     key = integer
2429
2245
        ...     key2 = float
2430
2246
        ...     [section]
2431
2247
        ...     key = string
2432
2248
        ...     key2 = float_list(4)
2433
2249
        ...        [[sub-section]]
2434
2250
        ...        key = option(option1, option2)
2435
2251
        ...        key2 = boolean'''.split('\\n')
2436
2252
        >>> val_test = ConfigObj(val_test_config, configspec=val_test_configspec)
2437
2253
        >>> val_test.validate(val)
2438
2254
        1
2439
2255
        >>> val_test['key'] = 'text not a digit'
2440
2256
        >>> val_res = val_test.validate(val)
2441
2257
        >>> val_res == {'key2': True, 'section': True, 'key': False}
2442
2258
        1
2443
2259
        >>> configspec = '''
2444
2260
        ...     test1=integer(30,50, default=40)
2445
2261
        ...     test2=string(default="hello")
2446
2262
        ...     test3=integer(default=3)
2447
2263
        ...     test4=float(6.0, default=6.0)
2448
2264
        ...     [section ]
2449
2265
        ...         test1=integer(30,50, default=40)
2450
2266
        ...         test2=string(default="hello")
2451
2267
        ...         test3=integer(default=3)
2452
2268
        ...         test4=float(6.0, default=6.0)
2453
2269
        ...         [[sub section]]
2454
2270
        ...             test1=integer(30,50, default=40)
2455
2271
        ...             test2=string(default="hello")
2456
2272
        ...             test3=integer(default=3)
2457
2273
        ...             test4=float(6.0, default=6.0)
2458
2274
        ...     '''.split('\\n')
2459
2275
        >>> default_test = ConfigObj(['test1=30'], configspec=configspec)
2460
2276
        >>> default_test
2461
2277
        {'test1': '30', 'section': {'sub section': {}}}
2462
2278
        >>> default_test.validate(val)
2463
2279
        1
2464
2280
        >>> default_test == {
2465
2281
        ...     'test1': 30,
2466
2282
        ...     'test2': 'hello',
2467
2283
        ...     'test3': 3,
2468
2284
        ...     'test4': 6.0,
2469
2285
        ...     'section': {
2470
2286
        ...         'test1': 40,
2471
2287
        ...         'test2': 'hello',
2472
2288
        ...         'test3': 3,
2473
2289
        ...         'test4': 6.0,
2474
2290
        ...         'sub section': {
2475
2291
        ...             'test1': 40,
2476
2292
        ...             'test3': 3,
2477
2293
        ...             'test2': 'hello',
2478
2294
        ...             'test4': 6.0,
2479
2295
        ...         },
2480
2296
        ...     },
2481
2297
        ... }
2482
2298
        1
2483
2299
2484
2300
        Now testing with repeated sections : BIG TEST
2485
2301
2486
2302
        >>> repeated_1 = '''
2487
2303
        ... [dogs]
2488
2304
        ...     [[__many__]] # spec for a dog
2489
2305
        ...         fleas = boolean(default=True)
2490
2306
        ...         tail = option(long, short, default=long)
2491
2307
        ...         name = string(default=rover)
2492
2308
        ...         [[[__many__]]]  # spec for a puppy
2493
2309
        ...             name = string(default="son of rover")
2494
2310
        ...             age = float(default=0.0)
2495
2311
        ... [cats]
2496
2312
        ...     [[__many__]] # spec for a cat
2497
2313
        ...         fleas = boolean(default=True)
2498
2314
        ...         tail = option(long, short, default=short)
2499
2315
        ...         name = string(default=pussy)
2500
2316
        ...         [[[__many__]]] # spec for a kitten
2501
2317
        ...             name = string(default="son of pussy")
2502
2318
        ...             age = float(default=0.0)
2503
2319
        ...         '''.split('\\n')
2504
2320
        >>> repeated_2 = '''
2505
2321
        ... [dogs]
2506
2322
        ...
2507
2323
        ...     # blank dogs with puppies
2508
2324
        ...     # should be filled in by the configspec
2509
2325
        ...     [[dog1]]
2510
2326
        ...         [[[puppy1]]]
2511
2327
        ...         [[[puppy2]]]
2512
2328
        ...         [[[puppy3]]]
2513
2329
        ...     [[dog2]]
2514
2330
        ...         [[[puppy1]]]
2515
2331
        ...         [[[puppy2]]]
2516
2332
        ...         [[[puppy3]]]
2517
2333
        ...     [[dog3]]
2518
2334
        ...         [[[puppy1]]]
2519
2335
        ...         [[[puppy2]]]
2520
2336
        ...         [[[puppy3]]]
2521
2337
        ... [cats]
2522
2338
        ...
2523
2339
        ...     # blank cats with kittens
2524
2340
        ...     # should be filled in by the configspec
2525
2341
        ...     [[cat1]]
2526
2342
        ...         [[[kitten1]]]
2527
2343
        ...         [[[kitten2]]]
2528
2344
        ...         [[[kitten3]]]
2529
2345
        ...     [[cat2]]
2530
2346
        ...         [[[kitten1]]]
2531
2347
        ...         [[[kitten2]]]
2532
2348
        ...         [[[kitten3]]]
2533
2349
        ...     [[cat3]]
2534
2350
        ...         [[[kitten1]]]
2535
2351
        ...         [[[kitten2]]]
2536
2352
        ...         [[[kitten3]]]
2537
2353
        ... '''.split('\\n')
2538
2354
        >>> repeated_3 = '''
2539
2355
        ... [dogs]
2540
2356
        ...
2541
2357
        ...     [[dog1]]
2542
2358
        ...     [[dog2]]
2543
2359
        ...     [[dog3]]
2544
2360
        ... [cats]
2545
2361
        ...
2546
2362
        ...     [[cat1]]
2547
2363
        ...     [[cat2]]
2548
2364
        ...     [[cat3]]
2549
2365
        ... '''.split('\\n')
2550
2366
        >>> repeated_4 = '''
2551
2367
        ... [__many__]
2552
2368
        ...
2553
2369
        ...     name = string(default=Michael)
2554
2370
        ...     age = float(default=0.0)
2555
2371
        ...     sex = option(m, f, default=m)
2556
2372
        ... '''.split('\\n')
2557
2373
        >>> repeated_5 = '''
2558
2374
        ... [cats]
2559
2375
        ... [[__many__]]
2560
2376
        ...     fleas = boolean(default=True)
2561
2377
        ...     tail = option(long, short, default=short)
2562
2378
        ...     name = string(default=pussy)
2563
2379
        ...     [[[description]]]
2564
2380
        ...         height = float(default=3.3)
2565
2381
        ...         weight = float(default=6)
2566
2382
        ...         [[[[coat]]]]
2567
2383
        ...             fur = option(black, grey, brown, "tortoise shell", default=black)
2568
2384
        ...             condition = integer(0,10, default=5)
2569
2385
        ... '''.split('\\n')
2570
2386
        >>> from validate import Validator
2571
2387
        >>> val= Validator()
2572
2388
        >>> repeater = ConfigObj(repeated_2, configspec=repeated_1)
2573
2389
        >>> repeater.validate(val)
2574
2390
        1
2575
2391
        >>> repeater == {
2576
2392
        ...     'dogs': {
2577
2393
        ...         'dog1': {
2578
2394
        ...             'fleas': True,
2579
2395
        ...             'tail': 'long',
2580
2396
        ...             'name': 'rover',
2581
2397
        ...             'puppy1': {'name': 'son of rover', 'age': 0.0},
2582
2398
        ...             'puppy2': {'name': 'son of rover', 'age': 0.0},
2583
2399
        ...             'puppy3': {'name': 'son of rover', 'age': 0.0},
2584
2400
        ...         },
2585
2401
        ...         'dog2': {
2586
2402
        ...             'fleas': True,
2587
2403
        ...             'tail': 'long',
2588
2404
        ...             'name': 'rover',
2589
2405
        ...             'puppy1': {'name': 'son of rover', 'age': 0.0},
2590
2406
        ...             'puppy2': {'name': 'son of rover', 'age': 0.0},
2591
2407
        ...             'puppy3': {'name': 'son of rover', 'age': 0.0},
2592
2408
        ...         },
2593
2409
        ...         'dog3': {
2594
2410
        ...             'fleas': True,
2595
2411
        ...             'tail': 'long',
2596
2412
        ...             'name': 'rover',
2597
2413
        ...             'puppy1': {'name': 'son of rover', 'age': 0.0},
2598
2414
        ...             'puppy2': {'name': 'son of rover', 'age': 0.0},
2599
2415
        ...             'puppy3': {'name': 'son of rover', 'age': 0.0},
2600
2416
        ...         },
2601
2417
        ...     },
2602
2418
        ...     'cats': {
2603
2419
        ...         'cat1': {
2604
2420
        ...             'fleas': True,
2605
2421
        ...             'tail': 'short',
2606
2422
        ...             'name': 'pussy',
2607
2423
        ...             'kitten1': {'name': 'son of pussy', 'age': 0.0},
2608
2424
        ...             'kitten2': {'name': 'son of pussy', 'age': 0.0},
2609
2425
        ...             'kitten3': {'name': 'son of pussy', 'age': 0.0},
2610
2426
        ...         },
2611
2427
        ...         'cat2': {
2612
2428
        ...             'fleas': True,
2613
2429
        ...             'tail': 'short',
2614
2430
        ...             'name': 'pussy',
2615
2431
        ...             'kitten1': {'name': 'son of pussy', 'age': 0.0},
2616
2432
        ...             'kitten2': {'name': 'son of pussy', 'age': 0.0},
2617
2433
        ...             'kitten3': {'name': 'son of pussy', 'age': 0.0},
2618
2434
        ...         },
2619
2435
        ...         'cat3': {
2620
2436
        ...             'fleas': True,
2621
2437
        ...             'tail': 'short',
2622
2438
        ...             'name': 'pussy',
2623
2439
        ...             'kitten1': {'name': 'son of pussy', 'age': 0.0},
2624
2440
        ...             'kitten2': {'name': 'son of pussy', 'age': 0.0},
2625
2441
        ...             'kitten3': {'name': 'son of pussy', 'age': 0.0},
2626
2442
        ...         },
2627
2443
        ...     },
2628
2444
        ... }
2629
2445
        1
2630
2446
        >>> repeater = ConfigObj(repeated_3, configspec=repeated_1)
2631
2447
        >>> repeater.validate(val)
2632
2448
        1
2633
2449
        >>> repeater == {
2634
2450
        ...     'cats': {
2635
2451
        ...         'cat1': {'fleas': True, 'tail': 'short', 'name': 'pussy'},
2636
2452
        ...         'cat2': {'fleas': True, 'tail': 'short', 'name': 'pussy'},
2637
2453
        ...         'cat3': {'fleas': True, 'tail': 'short', 'name': 'pussy'},
2638
2454
        ...     },
2639
2455
        ...     'dogs': {
2640
2456
        ...         'dog1': {'fleas': True, 'tail': 'long', 'name': 'rover'},
2641
2457
        ...         'dog2': {'fleas': True, 'tail': 'long', 'name': 'rover'},
2642
2458
        ...         'dog3': {'fleas': True, 'tail': 'long', 'name': 'rover'},
2643
2459
        ...     },
2644
2460
        ... }
2645
2461
        1
2646
2462
        >>> repeater = ConfigObj(configspec=repeated_4)
2647
2463
        >>> repeater['Michael'] = {}
2648
2464
        >>> repeater.validate(val)
2649
2465
        1
2650
2466
        >>> repeater == {
2651
2467
        ...     'Michael': {'age': 0.0, 'name': 'Michael', 'sex': 'm'},
2652
2468
        ... }
2653
2469
        1
2654
2470
        >>> repeater = ConfigObj(repeated_3, configspec=repeated_5)
2655
2471
        >>> repeater == {
2656
2472
        ...     'dogs': {'dog1': {}, 'dog2': {}, 'dog3': {}},
2657
2473
        ...     'cats': {'cat1': {}, 'cat2': {}, 'cat3': {}},
2658
2474
        ... }
2659
2475
        1
2660
2476
        >>> repeater.validate(val)
2661
2477
        1
2662
2478
        >>> repeater == {
2663
2479
        ...     'dogs': {'dog1': {}, 'dog2': {}, 'dog3': {}},
2664
2480
        ...     'cats': {
2665
2481
        ...         'cat1': {
2666
2482
        ...             'fleas': True,
2667
2483
        ...             'tail': 'short',
2668
2484
        ...             'name': 'pussy',
2669
2485
        ...             'description': {
2670
2486
        ...                 'weight': 6.0,
2671
2487
        ...                 'height': 3.2999999999999998,
2672
2488
        ...                 'coat': {'fur': 'black', 'condition': 5},
2673
2489
        ...             },
2674
2490
        ...         },
2675
2491
        ...         'cat2': {
2676
2492
        ...             'fleas': True,
2677
2493
        ...             'tail': 'short',
2678
2494
        ...             'name': 'pussy',
2679
2495
        ...             'description': {
2680
2496
        ...                 'weight': 6.0,
2681
2497
        ...                 'height': 3.2999999999999998,
2682
2498
        ...                 'coat': {'fur': 'black', 'condition': 5},
2683
2499
        ...             },
2684
2500
        ...         },
2685
2501
        ...         'cat3': {
2686
2502
        ...             'fleas': True,
2687
2503
        ...             'tail': 'short',
2688
2504
        ...             'name': 'pussy',
2689
2505
        ...             'description': {
2690
2506
        ...                 'weight': 6.0,
2691
2507
        ...                 'height': 3.2999999999999998,
2692
2508
        ...                 'coat': {'fur': 'black', 'condition': 5},
2693
2509
        ...             },
2694
2510
        ...         },
2695
2511
        ...     },
2696
2512
        ... }
2697
2513
        1
2698
2514
2699
2515
        Test that interpolation is preserved for validated string values.
2700
2516
        Also check that interpolation works in configspecs.
2701
2517
        >>> t = ConfigObj()
2702
2518
        >>> t['DEFAULT'] = {}
2703
2519
        >>> t['DEFAULT']['test'] = 'a'
2704
2520
        >>> t['test'] = '%(test)s'
2705
2521
        >>> t['test']
2706
2522
        'a'
2707
2523
        >>> v = Validator()
2708
2524
        >>> t.configspec = {'test': 'string'}
2709
2525
        >>> t.validate(v)
2710
2526
        1
2711
2527
        >>> t.interpolation = False
2712
2528
        >>> t
2713
2529
        {'test': '%(test)s', 'DEFAULT': {'test': 'a'}}
2714
2530
        >>> specs = [
2715
2531
        ...    'interpolated string  = string(default="fuzzy-%(man)s")',
2716
2532
        ...    '[DEFAULT]',
2717
2533
        ...    'man = wuzzy',
2718
2534
        ...    ]
2719
2535
        >>> c = ConfigObj(configspec=specs)
2720
2536
        >>> c.validate(v)
2721
2537
        1
2722
2538
        >>> c['interpolated string']
2723
2539
        'fuzzy-wuzzy'
2724
2540
2725
2541
        FIXME: Above tests will fail if we couldn't import Validator (the ones
2726
2542
        that don't raise errors will produce different output and still fail as
2727
2543
        tests)
2728
2544
2729
2545
        Test
2730
2546
        """
2731
2547
        if section is None:
2732
2548
            if self.configspec is None:
2733
2549
                raise ValueError, 'No configspec supplied.'
2734
2550
            if preserve_errors:
2735
2551
                if VdtMissingValue is None:
2736
2552
                    raise ImportError('Missing validate module.')
2737
2553
            section = self
2738
2554
        #
2739
2555
        spec_section = section.configspec
2740
2556
        if copy and hasattr(section, '_configspec_initial_comment'):
2741
2557
            section.initial_comment = section._configspec_initial_comment
2742
2558
            section.final_comment = section._configspec_final_comment
2743
2559
            section.encoding = section._configspec_encoding
2744
2560
            section.BOM = section._configspec_BOM
2745
2561
            section.newlines = section._configspec_newlines
2746
2562
            section.indent_type = section._configspec_indent_type
2747
2563
        if '__many__' in section.configspec:
2748
2564
            many = spec_section['__many__']
2749
2565
            # dynamically assign the configspecs
2750
2566
            # for the sections below
2751
2567
            for entry in section.sections:
2752
2568
                self._handle_repeat(section[entry], many)
2753
2569
        #
2754
2570
        out = {}
2755
2571
        ret_true = True
2756
2572
        ret_false = True
2757
2573
        order = [k for k in section._order if k in spec_section]
2758
2574
        order += [k for k in spec_section if k not in order]
2759
2575
        for entry in order:
2760
2576
            if entry == '__many__':
2761
2577
                continue
2762
2578
            if (not entry in section.scalars) or (entry in section.defaults):
2763
2579
                # missing entries
2764
2580
                # or entries from defaults
2765
2581
                missing = True
2766
2582
                val = None
2767
2583
                if copy and not entry in section.scalars:
2768
2584
                    # copy comments
2769
2585
                    section.comments[entry] = (
2770
2586
                        section._configspec_comments.get(entry, []))
2771
2587
                    section.inline_comments[entry] = (
2772
2588
                        section._configspec_inline_comments.get(entry, ''))
2773
2589
                #
2774
2590
            else:
2775
2591
                missing = False
2776
2592
                val = section[entry]
2777
2593
            try:
2778
2594
                check = validator.check(spec_section[entry],
2779
2595
                                        val,
2780
2596
                                        missing=missing
2781
2597
                                        )
2782
2598
            except validator.baseErrorClass, e:
2783
2599
                if not preserve_errors or isinstance(e, VdtMissingValue):
2784
2600
                    out[entry] = False
2785
2601
                else:
2786
2602
                    # preserve the error
2787
2603
                    out[entry] = e
2788
2604
                    ret_false = False
2789
2605
                ret_true = False
2790
2606
            else:
2791
2607
                ret_false = False
2792
2608
                out[entry] = True
2793
2609
                if self.stringify or missing:
2794
2610
                    # if we are doing type conversion
2795
2611
                    # or the value is a supplied default
2796
2612
                    if not self.stringify:
2797
2613
                        if isinstance(check, (list, tuple)):
2798
2614
                            # preserve lists
2799
2615
                            check = [self._str(item) for item in check]
2800
2616
                        elif missing and check is None:
2801
2617
                            # convert the None from a default to a ''
2802
2618
                            check = ''
2803
2619
                        else:
2804
2620
                            check = self._str(check)
2805
2621
                    if (check != val) or missing:
2806
2622
                        section[entry] = check
2807
2623
                if not copy and missing and entry not in section.defaults:
2808
2624
                    section.defaults.append(entry)
2809
2625
        #
2810
2626
        # Missing sections will have been created as empty ones when the
2811
2627
        # configspec was read.
2812
2628
        for entry in section.sections:
2813
2629
            # FIXME: this means DEFAULT is not copied in copy mode
2814
2630
            if section is self and entry == 'DEFAULT':
2815
2631
                continue
2816
2632
            if copy:
2817
2633
                section.comments[entry] = section._cs_section_comments[entry]
2818
2634
                section.inline_comments[entry] = (
2819
2635
                    section._cs_section_inline_comments[entry])
2820
2636
            check = self.validate(validator, preserve_errors=preserve_errors,
2821
2637
                copy=copy, section=section[entry])
2822
2638
            out[entry] = check
2823
2639
            if check == False:
2824
2640
                ret_true = False
2825
2641
            elif check == True:
2826
2642
                ret_false = False
2827
2643
            else:
2828
2644
                ret_true = False
2829
2645
                ret_false = False
2830
2646
        #
2831
2647
        if ret_true:
2832
2648
            return True
2833
2649
        elif ret_false:
2834
2650
            return False
2835
2651
        else:
2836
2652
            return out
2837
2653
2838
2654
2839
2655
class SimpleVal(object):
2840
2656
    """
2841
2657
    A simple validator.
2842
2658
    Can be used to check that all members expected are present.
2843
2659
2844
2660
    To use it, provide a configspec with all your members in (the value given
2845
2661
    will be ignored). Pass an instance of ``SimpleVal`` to the ``validate``
2846
2662
    method of your ``ConfigObj``. ``validate`` will return ``True`` if all
2847
2663
    members are present, or a dictionary with True/False meaning
2848
2664
    present/missing. (Whole missing sections will be replaced with ``False``)
2849
2665
2850
2666
    >>> val = SimpleVal()
2851
2667
    >>> config = '''
2852
2668
    ... test1=40
2853
2669
    ... test2=hello
2854
2670
    ... test3=3
2855
2671
    ... test4=5.0
2856
2672
    ... [section]
2857
2673
    ... test1=40
2858
2674
    ... test2=hello
2859
2675
    ... test3=3
2860
2676
    ... test4=5.0
2861
2677
    ...     [[sub section]]
2862
2678
    ...     test1=40
2863
2679
    ...     test2=hello
2864
2680
    ...     test3=3
2865
2681
    ...     test4=5.0
2866
2682
    ... '''.split('\\n')
2867
2683
    >>> configspec = '''
2868
2684
    ... test1=''
2869
2685
    ... test2=''
2870
2686
    ... test3=''
2871
2687
    ... test4=''
2872
2688
    ... [section]
2873
2689
    ... test1=''
2874
2690
    ... test2=''
2875
2691
    ... test3=''
2876
2692
    ... test4=''
2877
2693
    ...     [[sub section]]
2878
2694
    ...     test1=''
2879
2695
    ...     test2=''
2880
2696
    ...     test3=''
2881
2697
    ...     test4=''
2882
2698
    ... '''.split('\\n')
2883
2699
    >>> o = ConfigObj(config, configspec=configspec)
2884
2700
    >>> o.validate(val)
2885
2701
    1
2886
2702
    >>> o = ConfigObj(configspec=configspec)
2887
2703
    >>> o.validate(val)
2888
2704
    0
2889
2705
    """
2890
2706
2891
2707
    def __init__(self):
2892
2708
        self.baseErrorClass = ConfigObjError
2893
2709
2894
2710
    def check(self, check, member, missing=False):
2895
2711
        """A dummy check method, always returns the value unchanged."""
2896
2712
        if missing:
2897
2713
            raise self.baseErrorClass
2898
2714
        return member
2899
2715
2900
2716
# Check / processing functions for options
2901
2717
def flatten_errors(cfg, res, levels=None, results=None):
2902
2718
    """
2903
2719
    An example function that will turn a nested dictionary of results
2904
2720
    (as returned by ``ConfigObj.validate``) into a flat list.
2905
2721
2906
2722
    ``cfg`` is the ConfigObj instance being checked, ``res`` is the results
2907
2723
    dictionary returned by ``validate``.
2908
2724
2909
2725
    (This is a recursive function, so you shouldn't use the ``levels`` or
2910
2726
    ``results`` arguments - they are used by the function.
2911
2727
2912
2728
    Returns a list of keys that failed. Each member of the list is a tuple :
2913
2729
    ::
2914
2730
2915
2731
        ([list of sections...], key, result)
2916
2732
2917
2733
    If ``validate`` was called with ``preserve_errors=False`` (the default)
2918
2734
    then ``result`` will always be ``False``.
2919
2735
2920
2736
    *list of sections* is a flattened list of sections that the key was found
2921
2737
    in.
2922
2738
2923
2739
    If the section was missing then key will be ``None``.
2924
2740
2925
2741
    If the value (or section) was missing then ``result`` will be ``False``.
2926
2742
2927
2743
    If ``validate`` was called with ``preserve_errors=True`` and a value
2928
2744
    was present, but failed the check, then ``result`` will be the exception
2929
2745
    object returned. You can use this as a string that describes the failure.
2930
2746
2931
2747
    For example *The value "3" is of the wrong type*.
2932
2748
2933
2749
    # FIXME: is the ordering of the output arbitrary ?
2934
2750
    >>> import validate
2935
2751
    >>> vtor = validate.Validator()
2936
2752
    >>> my_ini = '''
2937
2753
    ...     option1 = True
2938
2754
    ...     [section1]
2939
2755
    ...     option1 = True
2940
2756
    ...     [section2]
2941
2757
    ...     another_option = Probably
2942
2758
    ...     [section3]
2943
2759
    ...     another_option = True
2944
2760
    ...     [[section3b]]
2945
2761
    ...     value = 3
2946
2762
    ...     value2 = a
2947
2763
    ...     value3 = 11
2948
2764
    ...     '''
2949
2765
    >>> my_cfg = '''
2950
2766
    ...     option1 = boolean()
2951
2767
    ...     option2 = boolean()
2952
2768
    ...     option3 = boolean(default=Bad_value)
2953
2769
    ...     [section1]
2954
2770
    ...     option1 = boolean()
2955
2771
    ...     option2 = boolean()
2956
2772
    ...     option3 = boolean(default=Bad_value)
2957
2773
    ...     [section2]
2958
2774
    ...     another_option = boolean()
2959
2775
    ...     [section3]
2960
2776
    ...     another_option = boolean()
2961
2777
    ...     [[section3b]]
2962
2778
    ...     value = integer
2963
2779
    ...     value2 = integer
2964
2780
    ...     value3 = integer(0, 10)
2965
2781
    ...         [[[section3b-sub]]]
2966
2782
    ...         value = string
2967
2783
    ...     [section4]
2968
2784
    ...     another_option = boolean()
2969
2785
    ...     '''
2970
2786
    >>> cs = my_cfg.split('\\n')
2971
2787
    >>> ini = my_ini.split('\\n')
2972
2788
    >>> cfg = ConfigObj(ini, configspec=cs)
2973
2789
    >>> res = cfg.validate(vtor, preserve_errors=True)
2974
2790
    >>> errors = []
2975
2791
    >>> for entry in flatten_errors(cfg, res):
2976
2792
    ...     section_list, key, error = entry
2977
2793
    ...     section_list.insert(0, '[root]')
2978
2794
    ...     if key is not None:
2979
2795
    ...        section_list.append(key)
2980
2796
    ...     else:
2981
2797
    ...         section_list.append('[missing]')
2982
2798
    ...     section_string = ', '.join(section_list)
2983
2799
    ...     errors.append((section_string, ' = ', error))
2984
2800
    >>> errors.sort()
2985
2801
    >>> for entry in errors:
2986
2802
    ...     print entry[0], entry[1], (entry[2] or 0)
2987
2803
    [root], option2  =  0
2988
2804
    [root], option3  =  the value "Bad_value" is of the wrong type.
2989
2805
    [root], section1, option2  =  0
2990
2806
    [root], section1, option3  =  the value "Bad_value" is of the wrong type.
2991
2807
    [root], section2, another_option  =  the value "Probably" is of the wrong type.
2992
2808
    [root], section3, section3b, section3b-sub, [missing]  =  0
2993
2809
    [root], section3, section3b, value2  =  the value "a" is of the wrong type.
2994
2810
    [root], section3, section3b, value3  =  the value "11" is too big.
2995
2811
    [root], section4, [missing]  =  0
2996
2812
    """
2997
2813
    if levels is None:
2998
2814
        # first time called
2999
2815
        levels = []
3000
2816
        results = []
3001
2817
    if res is True:
3002
2818
        return results
3003
2819
    if res is False:
3004
2820
        results.append((levels[:], None, False))
3005
2821
        if levels:
3006
2822
            levels.pop()
3007
2823
        return results
3008
2824
    for (key, val) in res.items():
3009
2825
        if val == True:
3010
2826
            continue
3011
2827
        if isinstance(cfg.get(key), dict):
3012
2828
            # Go down one level
3013
2829
            levels.append(key)
3014
2830
            flatten_errors(cfg[key], val, levels, results)
3015
2831
            continue
3016
2832
        results.append((levels[:], key, val))
3017
2833
    #
3018
2834
    # Go up one level
3019
2835
    if levels:
3020
2836
        levels.pop()
3021
2837
    #
3022
2838
    return results
3023
2839
3024
2840
3025
2841
# FIXME: test error code for badly built multiline values
3026
2842
# FIXME: test handling of StringIO
3027
2843
# FIXME: test interpolation with writing
3028
2844
3029
2845
def _doctest():
3030
2846
    """
3031
2847
    Dummy function to hold some of the doctests.
3032
2848
3033
2849
    >>> a.depth
3034
2850
    0
3035
2851
    >>> a == {
3036
2852
    ...     'key2': 'val',
3037
2853
    ...     'key1': 'val',
3038
2854
    ...     'lev1c': {
3039
2855
    ...         'lev2c': {
3040
2856
    ...             'lev3c': {
3041
2857
    ...                 'key1': 'val',
3042
2858
    ...             },
3043
2859
    ...         },
3044
2860
    ...     },
3045
2861
    ...     'lev1b': {
3046
2862
    ...         'key2': 'val',
3047
2863
    ...         'key1': 'val',
3048
2864
    ...         'lev2ba': {
3049
2865
    ...             'key1': 'val',
3050
2866
    ...         },
3051
2867
    ...         'lev2bb': {
3052
2868
    ...             'key1': 'val',
3053
2869
    ...         },
3054
2870
    ...     },
3055
2871
    ...     'lev1a': {
3056
2872
    ...         'key2': 'val',
3057
2873
    ...         'key1': 'val',
3058
2874
    ...     },
3059
2875
    ... }
3060
2876
    1
3061
2877
    >>> b.depth
3062
2878
    0
3063
2879
    >>> b == {
3064
2880
    ...     'key3': 'val3',
3065
2881
    ...     'key2': 'val2',
3066
2882
    ...     'key1': 'val1',
3067
2883
    ...     'section 1': {
3068
2884
    ...         'keys11': 'val1',
3069
2885
    ...         'keys13': 'val3',
3070
2886
    ...         'keys12': 'val2',
3071
2887
    ...     },
3072
2888
    ...     'section 2': {
3073
2889
    ...         'section 2 sub 1': {
3074
2890
    ...             'fish': '3',
3075
2891
    ...     },
3076
2892
    ...     'keys21': 'val1',
3077
2893
    ...     'keys22': 'val2',
3078
2894
    ...     'keys23': 'val3',
3079
2895
    ...     },
3080
2896
    ... }
3081
2897
    1
3082
2898
    >>> t = '''
3083
2899
    ... 'a' = b # !"$%^&*(),::;'@~#= 33
3084
2900
    ... "b" = b #= 6, 33
3085
2901
    ... ''' .split('\\n')
3086
2902
    >>> t2 = ConfigObj(t)
3087
2903
    >>> assert t2 == {'a': 'b', 'b': 'b'}
3088
2904
    >>> t2.inline_comments['b'] = ''
3089
2905
    >>> del t2['a']
3090
2906
    >>> assert t2.write() == ['','b = b', '']
3091
2907
3092
2908
    # Test ``list_values=False`` stuff
3093
2909
    >>> c = '''
3094
2910
    ...     key1 = no quotes
3095
2911
    ...     key2 = 'single quotes'
3096
2912
    ...     key3 = "double quotes"
3097
2913
    ...     key4 = "list", 'with', several, "quotes"
3098
2914
    ...     '''
3099
2915
    >>> cfg = ConfigObj(c.splitlines(), list_values=False)
3100
2916
    >>> cfg == {'key1': 'no quotes', 'key2': "'single quotes'",
3101
2917
    ... 'key3': '"double quotes"',
3102
2918
    ... 'key4': '"list", \\'with\\', several, "quotes"'
3103
2919
    ... }
3104
2920
    1
3105
2921
    >>> cfg = ConfigObj(list_values=False)
3106
2922
    >>> cfg['key1'] = 'Multiline\\nValue'
3107
2923
    >>> cfg['key2'] = '''"Value" with 'quotes' !'''
3108
2924
    >>> cfg.write()
3109
2925
    ["key1 = '''Multiline\\nValue'''", 'key2 = "Value" with \\'quotes\\' !']
3110
2926
    >>> cfg.list_values = True
3111
2927
    >>> cfg.write() == ["key1 = '''Multiline\\nValue'''",
3112
2928
    ... 'key2 = \\'\\'\\'"Value" with \\'quotes\\' !\\'\\'\\'']
3113
2929
    1
3114
2930
3115
2931
    Test flatten_errors:
3116
2932
3117
2933
    >>> from validate import Validator, VdtValueTooSmallError
3118
2934
    >>> config = '''
3119
2935
    ...     test1=40
3120
2936
    ...     test2=hello
3121
2937
    ...     test3=3
3122
2938
    ...     test4=5.0
3123
2939
    ...     [section]
3124
2940
    ...         test1=40
3125
2941
    ...         test2=hello
3126
2942
    ...         test3=3
3127
2943
    ...         test4=5.0
3128
2944
    ...         [[sub section]]
3129
2945
    ...             test1=40
3130
2946
    ...             test2=hello
3131
2947
    ...             test3=3
3132
2948
    ...             test4=5.0
3133
2949
    ... '''.split('\\n')
3134
2950
    >>> configspec = '''
3135
2951
    ...     test1= integer(30,50)
3136
2952
    ...     test2= string
3137
2953
    ...     test3=integer
3138
2954
    ...     test4=float(6.0)
3139
2955
    ...     [section ]
3140
2956
    ...         test1=integer(30,50)
3141
2957
    ...         test2=string
3142
2958
    ...         test3=integer
3143
2959
    ...         test4=float(6.0)
3144
2960
    ...         [[sub section]]
3145
2961
    ...             test1=integer(30,50)
3146
2962
    ...             test2=string
3147
2963
    ...             test3=integer
3148
2964
    ...             test4=float(6.0)
3149
2965
    ...     '''.split('\\n')
3150
2966
    >>> val = Validator()
3151
2967
    >>> c1 = ConfigObj(config, configspec=configspec)
3152
2968
    >>> res = c1.validate(val)
3153
2969
    >>> flatten_errors(c1, res) == [([], 'test4', False), (['section',
3154
2970
    ...     'sub section'], 'test4', False), (['section'], 'test4', False)]
3155
2971
    True
3156
2972
    >>> res = c1.validate(val, preserve_errors=True)
3157
2973
    >>> check = flatten_errors(c1, res)
3158
2974
    >>> check[0][:2]
3159
2975
    ([], 'test4')
3160
2976
    >>> check[1][:2]
3161
2977
    (['section', 'sub section'], 'test4')
3162
2978
    >>> check[2][:2]
3163
2979
    (['section'], 'test4')
3164
2980
    >>> for entry in check:
3165
2981
    ...     isinstance(entry[2], VdtValueTooSmallError)
3166
2982
    ...     print str(entry[2])
3167
2983
    True
3168
2984
    the value "5.0" is too small.
3169
2985
    True
3170
2986
    the value "5.0" is too small.
3171
2987
    True
3172
2988
    the value "5.0" is too small.
3173
2989
3174
2990
    Test unicode handling, BOM, write witha file like object and line endings :
3175
2991
    >>> u_base = '''
3176
2992
    ... # initial comment
3177
2993
    ...     # inital comment 2
3178
2994
    ...
3179
2995
    ... test1 = some value
3180
2996
    ... # comment
3181
2997
    ... test2 = another value    # inline comment
3182
2998
    ... # section comment
3183
2999
    ... [section]    # inline comment
3184
3000
    ...     test = test    # another inline comment
3185
3001
    ...     test2 = test2
3186
3002
    ...
3187
3003
    ... # final comment
3188
3004
    ... # final comment2
3189
3005
    ... '''
3190
3006
    >>> u = u_base.encode('utf_8').splitlines(True)
3191
3007
    >>> u[0] = BOM_UTF8 + u[0]
3192
3008
    >>> uc = ConfigObj(u)
3193
3009
    >>> uc.encoding = None
3194
3010
    >>> uc.BOM == True
3195
3011
    1
3196
3012
    >>> uc == {'test1': 'some value', 'test2': 'another value',
3197
3013
    ... 'section': {'test': 'test', 'test2': 'test2'}}
3198
3014
    1
3199
3015
    >>> uc = ConfigObj(u, encoding='utf_8', default_encoding='latin-1')
3200
3016
    >>> uc.BOM
3201
3017
    1
3202
3018
    >>> isinstance(uc['test1'], unicode)
3203
3019
    1
3204
3020
    >>> uc.encoding
3205
3021
    'utf_8'
3206
3022
    >>> uc.newlines
3207
3023
    '\\n'
3208
3024
    >>> uc['latin1'] = "This costs lot's of "
3209
3025
    >>> a_list = uc.write()
3210
3026
    >>> len(a_list)
3211
3027
    15
3212
3028
    >>> isinstance(a_list[0], str)
3213
3029
    1
3214
3030
    >>> a_list[0].startswith(BOM_UTF8)
3215
3031
    1
3216
3032
    >>> u = u_base.replace('\\n', '\\r\\n').encode('utf_8').splitlines(True)
3217
3033
    >>> uc = ConfigObj(u)
3218
3034
    >>> uc.newlines
3219
3035
    '\\r\\n'
3220
3036
    >>> uc.newlines = '\\r'
3221
3037
    >>> from cStringIO import StringIO
3222
3038
    >>> file_like = StringIO()
3223
3039
    >>> uc.write(file_like)
3224
3040
    >>> file_like.seek(0)
3225
3041
    >>> uc2 = ConfigObj(file_like)
3226
3042
    >>> uc2 == uc
3227
3043
    1
3228
3044
    >>> uc2.filename == None
3229
3045
    1
3230
3046
    >>> uc2.newlines == '\\r'
3231
3047
    1
3232
3048
3233
3049
    Test validate in copy mode
3234
3050
    >>> a = '''
3235
3051
    ... # Initial Comment
3236
3052
    ...
3237
3053
    ... key1 = string(default=Hello)    # comment 1
3238
3054
    ...
3239
3055
    ... # section comment
3240
3056
    ... [section] # inline comment
3241
3057
    ... # key1 comment
3242
3058
    ... key1 = integer(default=6) # an integer value
3243
3059
    ... # key2 comment
3244
3060
    ... key2 = boolean(default=True) # a boolean
3245
3061
    ...
3246
3062
    ...     # subsection comment
3247
3063
    ...     [[sub-section]] # inline comment
3248
3064
    ...     # another key1 comment
3249
3065
    ...     key1 = float(default=3.0) # a float'''.splitlines()
3250
3066
    >>> b = ConfigObj(configspec=a)
3251
3067
    >>> b.validate(val, copy=True)
3252
3068
    1
3253
3069
    >>> b.write() == ['',
3254
3070
    ... '# Initial Comment',
3255
3071
    ... '',
3256
3072
    ... 'key1 = Hello    # comment 1',
3257
3073
    ... '',
3258
3074
    ... '# section comment',
3259
3075
    ... '[section]    # inline comment',
3260
3076
    ... '    # key1 comment',
3261
3077
    ... '    key1 = 6    # an integer value',
3262
3078
    ... '    # key2 comment',
3263
3079
    ... '    key2 = True    # a boolean',
3264
3080
    ... '    ',
3265
3081
    ... '    # subsection comment',
3266
3082
    ... '    [[sub-section]]    # inline comment',
3267
3083
    ... '        # another key1 comment',
3268
3084
    ... '        key1 = 3.0    # a float']
3269
3085
    1
3270
3086
3271
3087
    Test Writing Empty Values
3272
3088
    >>> a = '''
3273
3089
    ...     key1 =
3274
3090
    ...     key2 =# a comment'''
3275
3091
    >>> b = ConfigObj(a.splitlines())
3276
3092
    >>> b.write()
3277
3093
    ['', 'key1 = ""', 'key2 = ""    # a comment']
3278
3094
    >>> b.write_empty_values = True
3279
3095
    >>> b.write()
3280
3096
    ['', 'key1 = ', 'key2 =     # a comment']
3281
3097
3282
3098
    Test unrepr when reading
3283
3099
    >>> a = '''
3284
3100
    ...     key1 = (1, 2, 3)    # comment
3285
3101
    ...     key2 = True
3286
3102
    ...     key3 = 'a string'
3287
3103
    ...     key4 = [1, 2, 3, 'a mixed list']
3288
3104
    ... '''.splitlines()
3289
3105
    >>> b = ConfigObj(a, unrepr=True)
3290
3106
    >>> b == {'key1': (1, 2, 3),
3291
3107
    ... 'key2': True,
3292
3108
    ... 'key3': 'a string',
3293
3109
    ... 'key4': [1, 2, 3, 'a mixed list']}
3294
3110
    1
3295
3111
3296
3112
    Test unrepr when writing
3297
3113
    >>> c = ConfigObj(b.write(), unrepr=True)
3298
3114
    >>> c == b
3299
3115
    1
3300
3116
3301
3117
    Test unrepr with multiline values
3302
3118
    >>> a = '''k = \"""{
3303
3119
    ... 'k1': 3,
3304
3120
    ... 'k2': 6.0}\"""
3305
3121
    ... '''.splitlines()
3306
3122
    >>> c = ConfigObj(a, unrepr=True)
3307
3123
    >>> c == {'k': {'k1': 3, 'k2': 6.0}}
3308
3124
    1
3309
3125
3310
3126
    Test unrepr with a dictionary
3311
3127
    >>> a = 'k = {"a": 1}'.splitlines()
3312
3128
    >>> c = ConfigObj(a, unrepr=True)
3313
3129
    >>> type(c['k']) == dict
3314
3130
    1
3315
3131
    """
3316
3132
3317
3133
if __name__ == '__main__':
3318
3134
    # run the code tests in doctest format
3319
3135
    #
3320
3136
    testconfig1 = """\
3321
3137
    key1= val    # comment 1
3322
3138
    key2= val    # comment 2
3323
3139
    # comment 3
3324
3140
    [lev1a]     # comment 4
3325
3141
    key1= val    # comment 5
3326
3142
    key2= val    # comment 6
3327
3143
    # comment 7
3328
3144
    [lev1b]    # comment 8
3329
3145
    key1= val    # comment 9
3330
3146
    key2= val    # comment 10
3331
3147
    # comment 11
3332
3148
        [[lev2ba]]    # comment 12
3333
3149
        key1= val    # comment 13
3334
3150
        # comment 14
3335
3151
        [[lev2bb]]    # comment 15
3336
3152
        key1= val    # comment 16
3337
3153
    # comment 17
3338
3154
    [lev1c]    # comment 18
3339
3155
    # comment 19
3340
3156
        [[lev2c]]    # comment 20
3341
3157
        # comment 21
3342
3158
            [[[lev3c]]]    # comment 22
3343
3159
            key1 = val    # comment 23"""
3344
3160
    #
3345
3161
    testconfig2 = """\
3346
3162
                        key1 = 'val1'
3347
3163
                        key2 =   "val2"
3348
3164
                        key3 = val3
3349
3165
                        ["section 1"] # comment
3350
3166
                        keys11 = val1
3351
3167
                        keys12 = val2
3352
3168
                        keys13 = val3
3353
3169
                        [section 2]
3354
3170
                        keys21 = val1
3355
3171
                        keys22 = val2
3356
3172
                        keys23 = val3
3357
3173
3358
3174
                            [['section 2 sub 1']]
3359
3175
                            fish = 3
3360
3176
    """
3361
3177
    #
3362
3178
    testconfig6 = '''
3363
3179
    name1 = """ a single line value """ # comment
3364
3180
    name2 = \''' another single line value \''' # comment
3365
3181
    name3 = """ a single line value """
3366
3182
    name4 = \''' another single line value \'''
3367
3183
        [ "multi section" ]
3368
3184
        name1 = """
3369
3185
        Well, this is a
3370
3186
        multiline value
3371
3187
        """
3372
3188
        name2 = \'''
3373
3189
        Well, this is a
3374
3190
        multiline value
3375
3191
        \'''
3376
3192
        name3 = """
3377
3193
        Well, this is a
3378
3194
        multiline value
3379
3195
        """     # a comment
3380
3196
        name4 = \'''
3381
3197
        Well, this is a
3382
3198
        multiline value
3383
3199
        \'''  # I guess this is a comment too
3384
3200
    '''
3385
3201
    #
3386
3202
    import doctest
3387
3203
    m = sys.modules.get('__main__')
3388
3204
    globs = m.__dict__.copy()
3389
3205
    a = ConfigObj(testconfig1.split('\n'), raise_errors=True)
3390
3206
    b = ConfigObj(testconfig2.split('\n'), raise_errors=True)
3391
3207
    i = ConfigObj(testconfig6.split('\n'), raise_errors=True)
3392
3208
    globs.update({
3393
3209
        'INTP_VER': INTP_VER,
3394
3210
        'a': a,
3395
3211
        'b': b,
3396
3212
        'i': i,
3397
3213
    })
3398
3214
    doctest.testmod(m, globs=globs)
3399
3215
3400
3216
"""
3401
3217
    BUGS
3402
3218
    ====
3403
3219
3404
3220
    None known.
3405
3221
3406
3222
    TODO
3407
3223
    ====
3408
3224
3409
3225
    Better support for configuration from multiple files, including tracking
3410
3226
    *where* the original file came from and writing changes to the correct
3411
3227
    file.
3412
3228
3413
3229
    Make ``newline`` an option (as well as an attribute) ?
3414
3230
3415
3231
    ``UTF16`` encoded files, when returned as a list of lines, will have the
3416
3232
    BOM at the start of every line. Should this be removed from all but the
3417
3233
    first line ?
3418
3234
3419
3235
    Option to set warning type for unicode decode ? (Defaults to strict).
3420
3236
3421
3237
    A method to optionally remove uniform indentation from multiline values.
3422
3238
    (do as an example of using ``walk`` - along with string-escape)
3423
3239
3424
3240
    Should the results dictionary from validate be an ordered dictionary if
3425
3241
    `odict <http://www.voidspace.org.uk/python/odict.html>`_ is available ?
3426
3242
3427
3243
    Implement a better ``__repr__`` ? (``ConfigObj({})``)
3428
3244
3429
3245
    Implement some of the sequence methods (which include slicing) from the
3430
3246
    newer ``odict`` ?
3431
3247
3432
3248
    ISSUES
3433
3249
    ======
3434
3250
3435
3251
    There is currently no way to specify the encoding of a configspec.
3436
3252
3437
3253
    When using ``copy`` mode for validation, it won't copy ``DEFAULT``
3438
3254
    sections. This is so that you *can* use interpolation in configspec
3439
3255
    files.
3440
3256
3441
3257
    ``validate`` doesn't report *extra* values or sections.
3442
3258
3443
3259
    You can't have a keyword with the same name as a section (in the same
3444
3260
    section). They are both dictionary keys - so they would overlap.
3445
3261
3446
3262
    ConfigObj doesn't quote and unquote values if ``list_values=False``.
3447
3263
    This means that leading or trailing whitespace in values will be lost when
3448
3264
    writing. (Unless you manually quote).
3449
3265
3450
3266
    Interpolation checks first the 'DEFAULT' subsection of the current
3451
3267
    section, next it checks the 'DEFAULT' section of the parent section,
3452
3268
    last it checks the 'DEFAULT' section of the main section.
3453
3269
3454
3270
    Logically a 'DEFAULT' section should apply to all subsections of the *same
3455
3271
    parent* - this means that checking the 'DEFAULT' subsection in the
3456
3272
    *current section* is not necessarily logical ?
3457
3273
3458
3274
    Does it matter that we don't support the ':' divider, which is supported
3459
3275
    by ``ConfigParser`` ?
3460
3276
3461
3277
    String interpolation and validation don't play well together. When
3462
3278
    validation changes type it sets the value. This will correctly fetch the
3463
3279
    value using interpolation - but then overwrite the interpolation reference.
3464
3280
    If the value is unchanged by validation (it's a string) - but other types
3465
3281
    will be.
3466
3282
3467
3283
3468
3284
    List Value Syntax
3469
3285
    =================
3470
3286
3471
3287
    List values allow you to specify multiple values for a keyword. This
3472
3288
    maps to a list as the resulting Python object when parsed.
3473
3289
3474
3290
    The syntax for lists is easy. A list is a comma separated set of values.
3475
3291
    If these values contain quotes, the hash mark, or commas, then the values
3476
3292
    can be surrounded by quotes. e.g. : ::
3477
3293
3478
3294
        keyword = value1, 'value 2', "value 3"
3479
3295
3480
3296
    If a value needs to be a list, but only has one member, then you indicate
3481
3297
    this with a trailing comma. e.g. : ::
3482
3298
3483
3299
        keyword = "single value",
3484
3300
3485
3301
    If a value needs to be a list, but it has no members, then you indicate
3486
3302
    this with a single comma. e.g. : ::
3487
3303
3488
3304
        keyword = ,     # an empty list
3489
3305
3490
3306
    Using triple quotes it will be possible for single values to contain
3491
3307
    newlines and *both* single quotes and double quotes. Triple quotes aren't
3492
3308
    allowed in list values. This means that the members of list values can't
3493
3309
    contain carriage returns (or line feeds :-) or both quote values.
3494
3310
3495
3311
    CHANGELOG
3496
3312
    =========
3497
3313
3498
3314
    2006/03/20
3499
3315
    ----------
3500
3316
3501
3317
    Empty values are now valid syntax. They are read as an empty string ``''``.
3502
3318
    (``key =``, or ``key = # comment``.)
3503
3319
3504
3320
    ``validate`` now honours the order of the configspec.
3505
3321
3506
3322
    Added the ``copy`` mode to validate.
3507
3323
3508
3324
    Fixed bug where files written on windows could be given '\r\r\n' line
3509
3325
    terminators.
3510
3326
3511
3327
    Fixed bug where last occuring comment line could be interpreted as the
3512
3328
    final comment if the last line isn't terminated.
3513
3329
3514
3330
    Fixed bug where nested list values would be flattened when ``write`` is
3515
3331
    called. Now sub-lists have a string representation written instead.
3516
3332
3517
3333
    Deprecated ``encode`` and ``decode`` methods instead.
3518
3334
3519
3335
    You can now pass in a COnfigObj instance as a configspec (remember to read
3520
3336
    the file using ``list_values=False``).
3521
3337
3522
3338
    2006/02/04
3523
3339
    ----------
3524
3340
3525
3341
    Removed ``BOM_UTF8`` from ``__all__``.
3526
3342
3527
3343
    The ``BOM`` attribute has become a boolean. (Defaults to ``False``.) It can
3528
3344
    be ``True`` for the ``UTF16/UTF8`` encodings.
3529
3345
3530
3346
    File like objects no longer need a ``seek`` attribute.
3531
3347
3532
3348
    ConfigObj no longer keeps a reference to file like objects. Instead the
3533
3349
    ``write`` method takes a file like object as an optional argument. (Which
3534
3350
    will be used in preference of the ``filename`` attribute if htat exists as
3535
3351
    well.)
3536
3352
3537
3353
    Full unicode support added. New options/attributes ``encoding``,
3538
3354
    ``default_encoding``.
3539
3355
3540
3356
    utf16 files decoded to unicode.
3541
3357
3542
3358
    If ``BOM`` is ``True``, but no encoding specified, then the utf8 BOM is
3543
3359
    written out at the start of the file. (It will normally only be ``True`` if
3544
3360
    the utf8 BOM was found when the file was read.)
3545
3361
3546
3362
    File paths are *not* converted to absolute paths, relative paths will
3547
3363
    remain relative as the ``filename`` attribute.
3548
3364
3549
3365
    Fixed bug where ``final_comment`` wasn't returned if ``write`` is returning
3550
3366
    a list of lines.
3551
3367
3552
3368
    2006/01/31
3553
3369
    ----------
3554
3370
3555
3371
    Added ``True``, ``False``, and ``enumerate`` if they are not defined.
3556
3372
    (``True`` and ``False`` are needed for *early* versions of Python 2.2,
3557
3373
    ``enumerate`` is needed for all versions ofPython 2.2)
3558
3374
3559
3375
    Deprecated ``istrue``, replaced it with ``as_bool``.
3560
3376
3561
3377
    Added ``as_int`` and ``as_float``.
3562
3378
3563
3379
    utf8 and utf16 BOM handled in an endian agnostic way.
3564
3380
3565
3381
    2005/12/14
3566
3382
    ----------
3567
3383
3568
3384
    Validation no longer done on the 'DEFAULT' section (only in the root
3569
3385
    level). This allows interpolation in configspecs.
3570
3386
3571
3387
    Change in validation syntax implemented in validate 0.2.1
3572
3388
3573
3389
    4.1.0
3574
3390
3575
3391
    2005/12/10
3576
3392
    ----------
3577
3393
3578
3394
    Added ``merge``, a recursive update.
3579
3395
3580
3396
    Added ``preserve_errors`` to ``validate`` and the ``flatten_errors``
3581
3397
    example function.
3582
3398
3583
3399
    Thanks to Matthew Brett for suggestions and helping me iron out bugs.
3584
3400
3585
3401
    Fixed bug where a config file is *all* comment, the comment will now be
3586
3402
    ``initial_comment`` rather than ``final_comment``.
3587
3403
3588
3404
    2005/12/02
3589
3405
    ----------
3590
3406
3591
3407
    Fixed bug in ``create_empty``. Thanks to Paul Jimenez for the report.
3592
3408
3593
3409
    2005/11/04
3594
3410
    ----------
3595
3411
3596
3412
    Fixed bug in ``Section.walk`` when transforming names as well as values.
3597
3413
3598
3414
    Added the ``istrue`` method. (Fetches the boolean equivalent of a string
3599
3415
    value).
3600
3416
3601
3417
    Fixed ``list_values=False`` - they are now only quoted/unquoted if they
3602
3418
    are multiline values.
3603
3419
3604
3420
    List values are written as ``item, item`` rather than ``item,item``.
3605
3421
3606
3422
    4.0.1
3607
3423
3608
3424
    2005/10/09
3609
3425
    ----------
3610
3426
3611
3427
    Fixed typo in ``write`` method. (Testing for the wrong value when resetting
3612
3428
    ``interpolation``).
3613
3429
3614
3430
    4.0.0 Final
3615
3431
3616
3432
    2005/09/16
3617
3433
    ----------
3618
3434
3619
3435
    Fixed bug in ``setdefault`` - creating a new section *wouldn't* return
3620
3436
    a reference to the new section.
3621
3437
3622
3438
    2005/09/09
3623
3439
    ----------
3624
3440
3625
3441
    Removed ``PositionError``.
3626
3442
3627
3443
    Allowed quotes around keys as documented.
3628
3444
3629
3445
    Fixed bug with commas in comments. (matched as a list value)
3630
3446
3631
3447
    Beta 5
3632
3448
3633
3449
    2005/09/07
3634
3450
    ----------
3635
3451
3636
3452
    Fixed bug in initialising ConfigObj from a ConfigObj.
3637
3453
3638
3454
    Changed the mailing list address.
3639
3455
3640
3456
    Beta 4
3641
3457
3642
3458
    2005/09/03
3643
3459
    ----------
3644
3460
3645
3461
    Fixed bug in ``Section.__delitem__`` oops.
3646
3462
3647
3463
    2005/08/28
3648
3464
    ----------
3649
3465
3650
3466
    Interpolation is switched off before writing out files.
3651
3467
3652
3468
    Fixed bug in handling ``StringIO`` instances. (Thanks to report from
3653
3469
    "Gustavo Niemeyer" <gustavo@niemeyer.net>)
3654
3470
3655
3471
    Moved the doctests from the ``__init__`` method to a separate function.
3656
3472
    (For the sake of IDE calltips).
3657
3473
3658
3474
    Beta 3
3659
3475
3660
3476
    2005/08/26
3661
3477
    ----------
3662
3478
3663
3479
    String values unchanged by validation *aren't* reset. This preserves
3664
3480
    interpolation in string values.
3665
3481
3666
3482
    2005/08/18
3667
3483
    ----------
3668
3484
3669
3485
    None from a default is turned to '' if stringify is off - because setting
3670
3486
    a value to None raises an error.
3671
3487
3672
3488
    Version 4.0.0-beta2
3673
3489
3674
3490
    2005/08/16
3675
3491
    ----------
3676
3492
3677
3493
    By Nicola Larosa
3678
3494
3679
3495
    Actually added the RepeatSectionError class ;-)
3680
3496
3681
3497
    2005/08/15
3682
3498
    ----------
3683
3499
3684
3500
    If ``stringify`` is off - list values are preserved by the ``validate``
3685
3501
    method. (Bugfix)
3686
3502
3687
3503
    2005/08/14
3688
3504
    ----------
3689
3505
3690
3506
    By Michael Foord
3691
3507
3692
3508
    Fixed ``simpleVal``.
3693
3509
3694
3510
    Added ``RepeatSectionError`` error if you have additional sections in a
3695
3511
    section with a ``__many__`` (repeated) section.
3696
3512
3697
3513
    By Nicola Larosa
3698
3514
3699
3515
    Reworked the ConfigObj._parse, _handle_error and _multiline methods:
3700
3516
    mutated the self._infile, self._index and self._maxline attributes into
3701
3517
    local variables and method parameters
3702
3518
3703
3519
    Reshaped the ConfigObj._multiline method to better reflect its semantics
3704
3520
3705
3521
    Changed the "default_test" test in ConfigObj.validate to check the fix for
3706
3522
    the bug in validate.Validator.check
3707
3523
3708
3524
    2005/08/13
3709
3525
    ----------
3710
3526
3711
3527
    By Nicola Larosa
3712
3528
3713
3529
    Updated comments at top
3714
3530
3715
3531
    2005/08/11
3716
3532
    ----------
3717
3533
3718
3534
    By Michael Foord
3719
3535
3720
3536
    Implemented repeated sections.
3721
3537
3722
3538
    By Nicola Larosa
3723
3539
3724
3540
    Added test for interpreter version: raises RuntimeError if earlier than
3725
3541
    2.2
3726
3542
3727
3543
    2005/08/10
3728
3544
    ----------
3729
3545
3730
3546
    By Michael Foord
3731
3547
3732
3548
    Implemented default values in configspecs.
3733
3549
3734
3550
    By Nicola Larosa
3735
3551
3736
3552
    Fixed naked except: clause in validate that was silencing the fact
3737
3553
    that Python2.2 does not have dict.pop
3738
3554
3739
3555
    2005/08/08
3740
3556
    ----------
3741
3557
3742
3558
    By Michael Foord
3743
3559
3744
3560
    Bug fix causing error if file didn't exist.
3745
3561
3746
3562
    2005/08/07
3747
3563
    ----------
3748
3564
3749
3565
    By Nicola Larosa
3750
3566
3751
3567
    Adjusted doctests for Python 2.2.3 compatibility
3752
3568
3753
3569
    2005/08/04
3754
3570
    ----------
3755
3571
3756
3572
    By Michael Foord
3757
3573
3758
3574
    Added the inline_comments attribute
3759
3575
3760
3576
    We now preserve and rewrite all comments in the config file
3761
3577
3762
3578
    configspec is now a section attribute
3763
3579
3764
3580
    The validate method changes values in place
3765
3581
3766
3582
    Added InterpolationError
3767
3583
3768
3584
    The errors now have line number, line, and message attributes. This
3769
3585
    simplifies error handling
3770
3586
3771
3587
    Added __docformat__
3772
3588
3773
3589
    2005/08/03
3774
3590
    ----------
3775
3591
3776
3592
    By Michael Foord
3777
3593
3778
3594
    Fixed bug in Section.pop (now doesn't raise KeyError if a default value
3779
3595
    is specified)
3780
3596
3781
3597
    Replaced ``basestring`` with ``types.StringTypes``
3782
3598
3783
3599
    Removed the ``writein`` method
3784
3600
3785
3601
    Added __version__
3786
3602
3787
3603
    2005/07/29
3788
3604
    ----------
3789
3605
3790
3606
    By Nicola Larosa
3791
3607
3792
3608
    Indentation in config file is not significant anymore, subsections are
3793
3609
    designated by repeating square brackets
3794
3610
3795
3611
    Adapted all tests and docs to the new format
3796
3612
3797
3613
    2005/07/28
3798
3614
    ----------
3799
3615
3800
3616
    By Nicola Larosa
3801
3617
3802
3618
    Added more tests
3803
3619
3804
3620
    2005/07/23
3805
3621
    ----------
3806
3622
3807
3623
    By Nicola Larosa
3808
3624
3809
3625
    Reformatted final docstring in ReST format, indented it for easier folding
3810
3626
3811
3627
    Code tests converted to doctest format, and scattered them around
3812
3628
    in various docstrings
3813
3629
3814
3630
    Walk method rewritten using scalars and sections attributes
3815
3631
3816
3632
    2005/07/22
3817
3633
    ----------
3818
3634
3819
3635
    By Nicola Larosa
3820
3636
3821
3637
    Changed Validator and SimpleVal "test" methods to "check"
3822
3638
3823
3639
    More code cleanup
3824
3640
3825
3641
    2005/07/21
3826
3642
    ----------
3827
3643
3828
3644
    Changed Section.sequence to Section.scalars and Section.sections
3829
3645
3830
3646
    Added Section.configspec
3831
3647
3832
3648
    Sections in the root section now have no extra indentation
3833
3649
3834
3650
    Comments now better supported in Section and preserved by ConfigObj
3835
3651
3836
3652
    Comments also written out
3837
3653
3838
3654
    Implemented initial_comment and final_comment
3839
3655
3840
3656
    A scalar value after a section will now raise an error
3841
3657
3842
3658
    2005/07/20
3843
3659
    ----------
3844
3660
3845
3661
    Fixed a couple of bugs
3846
3662
3847
3663
    Can now pass a tuple instead of a list
3848
3664
3849
3665
    Simplified dict and walk methods
3850
3666
3851
3667
    Added __str__ to Section
3852
3668
3853
3669
    2005/07/10
3854
3670
    ----------
3855
3671
3856
3672
    By Nicola Larosa
3857
3673
3858
3674
    More code cleanup
3859
3675
3860
3676
    2005/07/08
3861
3677
    ----------
3862
3678
3863
3679
    The stringify option implemented. On by default.
3864
3680
3865
3681
    2005/07/07
3866
3682
    ----------
3867
3683
3868
3684
    Renamed private attributes with a single underscore prefix.
3869
3685
3870
3686
    Changes to interpolation - exceeding recursion depth, or specifying a
3871
3687
    missing value, now raise errors.
3872
3688
3873
3689
    Changes for Python 2.2 compatibility. (changed boolean tests - removed
3874
3690
    ``is True`` and ``is False``)
3875
3691
3876
3692
    Added test for duplicate section and member (and fixed bug)
3877
3693
3878
3694
    2005/07/06
3879
3695
    ----------
3880
3696
3881
3697
    By Nicola Larosa
3882
3698
3883
3699
    Code cleanup
3884
3700
3885
3701
    2005/07/02
3886
3702
    ----------
3887
3703
3888
3704
    Version 0.1.0
3889
3705
3890
3706
    Now properly handles values including comments and lists.
3891
3707
3892
3708
    Better error handling.
3893
3709
3894
3710
    String interpolation.
3895
3711
3896
3712
    Some options implemented.
3897
3713
3898
3714
    You can pass a Section a dictionary to initialise it.
3899
3715
3900
3716
    Setting a Section member to a dictionary will create a Section instance.
3901
3717
3902
3718
    2005/06/26
3903
3719
    ----------
3904
3720
3905
3721
    Version 0.0.1
3906
3722
3907
3723
    Experimental reader.
3908
3724
3909
3725
    A reasonably elegant implementation - a basic reader in 160 lines of code.
3910
3726
3911
3727
    *A programming language is a medium of expression.* - Paul Graham
3912
3728
"""
3913
3729
0
3914
=== modified file 'landscape/lib/persist.py'
3915
--- landscape/lib/persist.py	2011-07-12 09:34:49 +0000
3916
+++ landscape/lib/persist.py	2013-04-05 16:00:29 +0000
3917
@@ -24,7 +24,7 @@
3918
24
import re
24
import re
3919
25
25
3920
26
26
3922
27
__all__ = ["Persist", "PickleBackend", "BPickleBackend", "ConfigObjBackend",
27
__all__ = ["Persist", "PickleBackend", "BPickleBackend",
3923
28
           "path_string_to_tuple", "path_tuple_to_string", "RootedPersist",
28
           "path_string_to_tuple", "path_tuple_to_string", "RootedPersist",
3924
29
           "PersistError", "PersistReadOnlyError"]
29
           "PersistError", "PersistReadOnlyError"]
3925
30
30
3926
@@ -141,6 +141,9 @@
3927
141
141
3928
142
        If None is specified, then the filename passed during construction will
142
        If None is specified, then the filename passed during construction will
3929
143
        be used.
143
        be used.
3930
144
3931
145
        If the destination file already exists, it will be renamed
3932
146
        to C{<filepath>.old}.
3933
144
        """
147
        """
3934
145
        if filepath is None:
148
        if filepath is None:
3935
146
            if self.filename is None:
149
            if self.filename is None:
3936
@@ -414,6 +417,21 @@
3937
414
417
3938
415
418
3939
416
def path_string_to_tuple(path):
419
def path_string_to_tuple(path):
3940
420
    """Convert a L{Persist} path string to a path tuple.
3941
421
3942
422
    Examples:
3943
423
3944
424
        >>> path_string_to_tuple("ab")
3945
425
        ("ab",)
3946
426
        >>> path_string_to_tuple("ab.cd")
3947
427
        ("ab", "cd"))
3948
428
        >>> path_string_to_tuple("ab[0][1]")
3949
429
        ("ab", 0, 1)
3950
430
        >>> path_string_to_tuple("ab[0].cd[1]")
3951
431
        ("ab", 0, "cd", 1)
3952
432
3953
433
    Raises L{PersistError} if the given path string is invalid.
3954
434
    """
3955
417
    if "." not in path and "[" not in path:
435
    if "." not in path and "[" not in path:
3956
418
        return (path,)
436
        return (path,)
3957
419
    result = []
437
    result = []
3958
@@ -441,6 +459,35 @@
3959
441
459
3960
442
460
3961
443
class Backend(object):
461
class Backend(object):
3962
462
    """
3963
463
    Base class for L{Persist} backends implementing hierarchical storage
3964
464
    functionality.
3965
465
3966
466
    Each node of the hierarchy is an object of type C{dict}, C{list}
3967
467
    or C{tuple}. A node can have zero or more children, each child can be
3968
468
    another node or a leaf value compatible with the backend's serialization
3969
469
    mechanism.
3970
470
3971
471
    Each child element is associated with a unique key, that can be used to
3972
472
    get, set or remove the child itself from its containing node. If the node
3973
473
    object is of type C{dict}, then the child keys will be the keys of the
3974
474
    dictionary, otherwise if the node object is of type C{list} or C{tuple}
3975
475
    the child element keys are the indexes of the available items, or the value
3976
476
    of items theselves.
3977
477
3978
478
    The root node object is always a C{dict}.
3979
479
3980
480
    For example:
3981
481
3982
482
        >>> root = backend.new()
3983
483
        >>> backend.set(root, "foo", "bar")
3984
484
        'bar'
3985
485
        >>> egg = backend.set(root, "egg", [1, 2, 3])
3986
486
        >>> backend.set(egg, 0, 10)
3987
487
        10
3988
488
        >>> root
3989
489
        {'foo': 'bar', 'egg': [10, 2, 3]}
3990
490
    """
3991
444
491
3992
445
    def new(self):
492
    def new(self):
3993
446
        raise NotImplementedError
493
        raise NotImplementedError
3994
@@ -452,6 +499,7 @@
3995
452
        raise NotImplementedError
499
        raise NotImplementedError
3996
453
500
3997
454
    def get(self, obj, elem, _marker=NOTHING):
501
    def get(self, obj, elem, _marker=NOTHING):
3998
502
        """Lookup a child in the given node object."""
3999
455
        if type(obj) is dict:
503
        if type(obj) is dict:
4000
456
            newobj = obj.get(elem, _marker)
504
            newobj = obj.get(elem, _marker)
4001
457
        elif type(obj) in (tuple, list):
505
        elif type(obj) in (tuple, list):
4002
@@ -469,6 +517,7 @@
4003
469
        return newobj
517
        return newobj
4004
470
518
4005
471
    def set(self, obj, elem, value):
519
    def set(self, obj, elem, value):
4006
520
        """Set the value of the given child in the given node object."""
4007
472
        if type(obj) is dict:
521
        if type(obj) is dict:
4008
473
            newobj = obj[elem] = value
522
            newobj = obj[elem] = value
4009
474
        elif type(obj) is list and type(elem) is int:
523
        elif type(obj) is list and type(elem) is int:
4010
@@ -485,6 +534,12 @@
4011
485
        return newobj
534
        return newobj
4012
486
535
4013
487
    def remove(self, obj, elem, isvalue):
536
    def remove(self, obj, elem, isvalue):
4014
537
        """Remove a the given child in the given node object.
4015
538
4016
539
        @param isvalue: In case the node object is a C{list}, a boolean
4017
540
            indicating if C{elem} is the index of the child or the value
4018
541
            of the child itself.
4019
542
        """
4020
488
        result = False
543
        result = False
4021
489
        if type(obj) is dict:
544
        if type(obj) is dict:
4022
490
            if elem in obj:
545
            if elem in obj:
4023
@@ -505,20 +560,24 @@
4024
505
        return result
560
        return result
4025
506
561
4026
507
    def copy(self, value):
562
    def copy(self, value):
4027
563
        """Copy a node or a value."""
4028
508
        if type(value) in (dict, list):
564
        if type(value) in (dict, list):
4029
509
            return copy.deepcopy(value)
565
            return copy.deepcopy(value)
4030
510
        return value
566
        return value
4031
511
567
4032
512
    def empty(self, obj):
568
    def empty(self, obj):
4033
569
        """Whether the given node object has no children."""
4034
513
        return (not obj)
570
        return (not obj)
4035
514
571
4036
515
    def has(self, obj, elem):
572
    def has(self, obj, elem):
4037
573
        """Whether the given node object contains the given child element."""
4038
516
        contains = getattr(obj, "__contains__", None)
574
        contains = getattr(obj, "__contains__", None)
4039
517
        if contains:
575
        if contains:
4040
518
            return contains(elem)
576
            return contains(elem)
4041
519
        return NotImplemented
577
        return NotImplemented
4042
520
578
4043
521
    def keys(self, obj):
579
    def keys(self, obj):
4044
580
        """Return the keys of the child elements of the given node object."""
4045
522
        keys = getattr(obj, "keys", None)
581
        keys = getattr(obj, "keys", None)
4046
523
        if keys:
582
        if keys:
4047
524
            return keys()
583
            return keys()
4048
@@ -574,49 +633,4 @@
4049
574
        finally:
633
        finally:
4050
575
            file.close()
634
            file.close()
4051
576
635
4052
577
4053
578
class ConfigObjBackend(Backend):
4054
579
4055
580
    def __init__(self):
4056
581
        from landscape.lib import configobj
4057
582
        self.ConfigObj = configobj.ConfigObj
4058
583
        self.Section = configobj.Section
4059
584
4060
585
    def new(self):
4061
586
        return self.ConfigObj(unrepr=True)
4062
587
4063
588
    def load(self, filepath):
4064
589
        return self.ConfigObj(filepath, unrepr=True)
4065
590
4066
591
    def save(self, filepath, map):
4067
592
        file = open(filepath, "w")
4068
593
        try:
4069
594
            map.write(file)
4070
595
        finally:
4071
596
            file.close()
4072
597
4073
598
    def get(self, obj, elem, _marker=NOTHING):
4074
599
        if isinstance(obj, self.Section):
4075
600
            return obj.get(elem, _marker)
4076
601
        return Backend.get(self, obj, elem)
4077
602
4078
603
    def set(self, obj, elem, value):
4079
604
        if isinstance(obj, self.Section):
4080
605
            obj[elem] = value
4081
606
            return obj[elem]
4082
607
        return Backend.set(self, obj, elem, value)
4083
608
4084
609
    def remove(self, obj, elem, isvalue):
4085
610
        if isinstance(obj, self.Section):
4086
611
            if elem in obj:
4087
612
                del obj[elem]
4088
613
                return True
4089
614
            return False
4090
615
        return Backend.remove(self, obj, elem, isvalue)
4091
616
4092
617
    def copy(self, value):
4093
618
        if isinstance(value, self.Section):
4094
619
            return value.dict()
4095
620
        return Backend.copy(self, value)
4096
621
4097
622
# vim:ts=4:sw=4:et
636
# vim:ts=4:sw=4:et
4098
623
637
4099
=== modified file 'landscape/lib/tests/test_persist.py'
4100
--- landscape/lib/tests/test_persist.py	2011-07-12 09:34:49 +0000
4101
+++ landscape/lib/tests/test_persist.py	2013-04-05 16:00:29 +0000
4102
@@ -3,7 +3,7 @@
4103
3
3
4104
4
from landscape.lib.persist import (
4
from landscape.lib.persist import (
4105
5
    path_string_to_tuple, path_tuple_to_string, Persist, RootedPersist,
5
    path_string_to_tuple, path_tuple_to_string, Persist, RootedPersist,
4107
6
    PickleBackend, ConfigObjBackend, PersistError, PersistReadOnlyError)
6
    PickleBackend, PersistError, PersistReadOnlyError)
4108
7
from landscape.tests.helpers import LandscapeTest
7
from landscape.tests.helpers import LandscapeTest
4109
8
8
4110
9
9
4111
@@ -444,12 +444,6 @@
4112
444
        return Persist(PickleBackend(), *args, **kwargs)
444
        return Persist(PickleBackend(), *args, **kwargs)
4113
445
445
4114
446
446
4115
447
class ConfigObjPersistTest(GeneralPersistTest, SaveLoadPersistTest):
4116
448
4117
449
    def build_persist(self, *args, **kwargs):
4118
450
        return Persist(ConfigObjBackend(), *args, **kwargs)
4119
451
4120
452
4121
453
class RootedPersistTest(GeneralPersistTest):
447
class RootedPersistTest(GeneralPersistTest):
4122
454
448
4123
455
    def build_persist(self, *args, **kwargs):
449
    def build_persist(self, *args, **kwargs):
4124
456
450
4125
=== removed file 'landscape/lib/twisted_amp.py'
4126
--- landscape/lib/twisted_amp.py	2010-06-08 07:59:32 +0000
4127
+++ landscape/lib/twisted_amp.py	1970-01-01 00:00:00 +0000
4128
@@ -1,2223 +0,0 @@
4129
1
# -*- test-case-name: twisted.test.test_amp -*-
4130
2
# Copyright (c) 2005 Divmod, Inc.
4131
3
# Copyright (c) 2007 Twisted Matrix Laboratories.
4132
4
# See LICENSE for details.
4133
5
4134
6
"""
4135
7
This module implements AMP, the Asynchronous Messaging Protocol.
4136
8
4137
9
AMP is a protocol for sending multiple asynchronous request/response pairs over
4138
10
the same connection.  Requests and responses are both collections of key/value
4139
11
pairs.
4140
12
4141
13
AMP is a very simple protocol which is not an application.  This module is a
4142
14
"protocol construction kit" of sorts; it attempts to be the simplest wire-level
4143
15
implementation of Deferreds.  AMP provides the following base-level features:
4144
16
4145
17
    - Asynchronous request/response handling (hence the name)
4146
18
4147
19
    - Requests and responses are both key/value pairs
4148
20
4149
21
    - Binary transfer of all data: all data is length-prefixed.  Your
4150
22
      application will never need to worry about quoting.
4151
23
4152
24
    - Command dispatching (like HTTP Verbs): the protocol is extensible, and
4153
25
      multiple AMP sub-protocols can be grouped together easily.
4154
26
4155
27
The protocol implementation also provides a few additional features which are
4156
28
not part of the core wire protocol, but are nevertheless very useful:
4157
29
4158
30
    - Tight TLS integration, with an included StartTLS command.
4159
31
4160
32
    - Handshaking to other protocols: because AMP has well-defined message
4161
33
      boundaries and maintains all incoming and outgoing requests for you, you
4162
34
      can start a connection over AMP and then switch to another protocol.
4163
35
      This makes it ideal for firewall-traversal applications where you may
4164
36
      have only one forwarded port but multiple applications that want to use
4165
37
      it.
4166
38
4167
39
Using AMP with Twisted is simple.  Each message is a command, with a response.
4168
40
You begin by defining a command type.  Commands specify their input and output
4169
41
in terms of the types that they expect to see in the request and response
4170
42
key-value pairs.  Here's an example of a command that adds two integers, 'a'
4171
43
and 'b'::
4172
44
4173
45
    class Sum(amp.Command):
4174
46
        arguments = [('a', amp.Integer()),
4175
47
                     ('b', amp.Integer())]
4176
48
        response = [('total', amp.Integer())]
4177
49
4178
50
Once you have specified a command, you need to make it part of a protocol, and
4179
51
define a responder for it.  Here's a 'JustSum' protocol that includes a
4180
52
responder for our 'Sum' command::
4181
53
4182
54
    class JustSum(amp.AMP):
4183
55
        def sum(self, a, b):
4184
56
            total = a + b
4185
57
            print 'Did a sum: %d + %d = %d' % (a, b, total)
4186
58
            return {'total': total}
4187
59
        Sum.responder(sum)
4188
60
4189
61
Later, when you want to actually do a sum, the following expression will return
4190
62
a Deferred which will fire with the result::
4191
63
4192
64
    ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback(
4193
65
        lambda p: p.callRemote(Sum, a=13, b=81)).addCallback(
4194
66
            lambda result: result['total'])
4195
67
4196
68
You can also define the propagation of specific errors in AMP.  For example,
4197
69
for the slightly more complicated case of division, we might have to deal with
4198
70
division by zero::
4199
71
4200
72
    class Divide(amp.Command):
4201
73
        arguments = [('numerator', amp.Integer()),
4202
74
                     ('denominator', amp.Integer())]
4203
75
        response = [('result', amp.Float())]
4204
76
        errors = {ZeroDivisionError: 'ZERO_DIVISION'}
4205
77
4206
78
The 'errors' mapping here tells AMP that if a responder to Divide emits a
4207
79
L{ZeroDivisionError}, then the other side should be informed that an error of
4208
80
the type 'ZERO_DIVISION' has occurred.  Writing a responder which takes
4209
81
advantage of this is very simple - just raise your exception normally::
4210
82
4211
83
    class JustDivide(amp.AMP):
4212
84
        def divide(self, numerator, denominator):
4213
85
            result = numerator / denominator
4214
86
            print 'Divided: %d / %d = %d' % (numerator, denominator, total)
4215
87
            return {'result': result}
4216
88
        Divide.responder(divide)
4217
89
4218
90
On the client side, the errors mapping will be used to determine what the
4219
91
'ZERO_DIVISION' error means, and translated into an asynchronous exception,
4220
92
which can be handled normally as any L{Deferred} would be::
4221
93
4222
94
    def trapZero(result):
4223
95
        result.trap(ZeroDivisionError)
4224
96
        print "Divided by zero: returning INF"
4225
97
        return 1e1000
4226
98
    ClientCreator(reactor, amp.AMP).connectTCP(...).addCallback(
4227
99
        lambda p: p.callRemote(Divide, numerator=1234,
4228
100
                               denominator=0)
4229
101
        ).addErrback(trapZero)
4230
102
4231
103
For a complete, runnable example of both of these commands, see the files in
4232
104
the Twisted repository::
4233
105
4234
106
    doc/core/examples/ampserver.py
4235
107
    doc/core/examples/ampclient.py
4236
108
4237
109
On the wire, AMP is a protocol which uses 2-byte lengths to prefix keys and
4238
110
values, and empty keys to separate messages::
4239
111
4240
112
    <2-byte length><key><2-byte length><value>
4241
113
    <2-byte length><key><2-byte length><value>
4242
114
    ...
4243
115
    <2-byte length><key><2-byte length><value>
4244
116
    <NUL><NUL>                  # Empty Key == End of Message
4245
117
4246
118
And so on.  Because it's tedious to refer to lengths and NULs constantly, the
4247
119
documentation will refer to packets as if they were newline delimited, like
4248
120
so::
4249
121
4250
122
    C: _command: sum
4251
123
    C: _ask: ef639e5c892ccb54
4252
124
    C: a: 13
4253
125
    C: b: 81
4254
126
4255
127
    S: _answer: ef639e5c892ccb54
4256
128
    S: total: 94
4257
129
4258
130
Notes:
4259
131
4260
132
Values are limited to the maximum encodable size in a 16-bit length, 65535
4261
133
bytes.
4262
134
4263
135
Keys are limited to the maximum encodable size in a 8-bit length, 255 bytes.
4264
136
Note that we still use 2-byte lengths to encode keys.  This small redundancy
4265
137
has several features:
4266
138
4267
139
    - If an implementation becomes confused and starts emitting corrupt data,
4268
140
      or gets keys confused with values, many common errors will be
4269
141
      signalled immediately instead of delivering obviously corrupt packets.
4270
142
4271
143
    - A single NUL will separate every key, and a double NUL separates
4272
144
      messages.  This provides some redundancy when debugging traffic dumps.
4273
145
4274
146
    - NULs will be present at regular intervals along the protocol, providing
4275
147
      some padding for otherwise braindead C implementations of the protocol,
4276
148
      so that <stdio.h> string functions will see the NUL and stop.
4277
149
4278
150
    - This makes it possible to run an AMP server on a port also used by a
4279
151
      plain-text protocol, and easily distinguish between non-AMP clients (like
4280
152
      web browsers) which issue non-NUL as the first byte, and AMP clients,
4281
153
      which always issue NUL as the first byte.
4282
154
4283
155
"""
4284
156
4285
157
__metaclass__ = type
4286
158
4287
159
import types, warnings
4288
160
4289
161
from cStringIO import StringIO
4290
162
from struct import pack
4291
163
4292
164
from zope.interface import Interface, implements
4293
165
4294
166
from twisted.python.reflect import accumulateClassDict
4295
167
from twisted.python.failure import Failure
4296
168
from twisted.python import log, filepath
4297
169
4298
170
from twisted.internet.main import CONNECTION_LOST
4299
171
from twisted.internet.error import ConnectionLost
4300
172
from twisted.internet.defer import Deferred, maybeDeferred, fail
4301
173
from twisted.protocols.basic import Int16StringReceiver, StatefulStringProtocol
4302
174
4303
175
#from twisted.internet._sslverify import problemsFromTransport
4304
176
4305
177
# I'd like this to use the exposed public API, but for some reason, when it was
4306
178
# moved, these names were not exposed by internet.ssl.
4307
179
4308
180
#from twisted.internet.ssl import CertificateOptions, Certificate, DN, KeyPair
4309
181
4310
182
ASK = '_ask'
4311
183
ANSWER = '_answer'
4312
184
COMMAND = '_command'
4313
185
ERROR = '_error'
4314
186
ERROR_CODE = '_error_code'
4315
187
ERROR_DESCRIPTION = '_error_description'
4316
188
UNKNOWN_ERROR_CODE = 'UNKNOWN'
4317
189
UNHANDLED_ERROR_CODE = 'UNHANDLED'
4318
190
4319
191
MAX_KEY_LENGTH = 0xff
4320
192
MAX_VALUE_LENGTH = 0xffff
4321
193
4322
194
4323
195
class IBoxSender(Interface):
4324
196
    """
4325
197
    A transport which can send L{AmpBox} objects.
4326
198
    """
4327
199
4328
200
    def sendBox(box):
4329
201
        """
4330
202
        Send an L{AmpBox}.
4331
203
4332
204
        @raise ProtocolSwitched: if the underlying protocol has been
4333
205
        switched.
4334
206
4335
207
        @raise ConnectionLost: if the underlying connection has already been
4336
208
        lost.
4337
209
        """
4338
210
4339
211
    def unhandledError(failure):
4340
212
        """
4341
213
        An unhandled error occurred in response to a box.  Log it
4342
214
        appropriately.
4343
215
4344
216
        @param failure: a L{Failure} describing the error that occurred.
4345
217
        """
4346
218
4347
219
4348
220
4349
221
class IBoxReceiver(Interface):
4350
222
    """
4351
223
    An application object which can receive L{AmpBox} objects and dispatch them
4352
224
    appropriately.
4353
225
    """
4354
226
4355
227
    def startReceivingBoxes(boxSender):
4356
228
        """
4357
229
        The L{ampBoxReceived} method will start being called; boxes may be
4358
230
        responded to by responding to the given L{IBoxSender}.
4359
231
4360
232
        @param boxSender: an L{IBoxSender} provider.
4361
233
        """
4362
234
4363
235
4364
236
    def ampBoxReceived(box):
4365
237
        """
4366
238
        A box was received from the transport; dispatch it appropriately.
4367
239
        """
4368
240
4369
241
4370
242
    def stopReceivingBoxes(reason):
4371
243
        """
4372
244
        No further boxes will be received on this connection.
4373
245
4374
246
        @type reason: L{Failure}
4375
247
        """
4376
248
4377
249
4378
250
4379
251
class IResponderLocator(Interface):
4380
252
    """
4381
253
    An application object which can look up appropriate responder methods for
4382
254
    AMP commands.
4383
255
    """
4384
256
4385
257
    def locateResponder(self, name):
4386
258
        """
4387
259
        Locate a responder method appropriate for the named command.
4388
260
4389
261
        @param name: the wire-level name (commandName) of the AMP command to be
4390
262
        responded to.
4391
263
4392
264
        @return: a 1-argument callable that takes an L{AmpBox} with argument
4393
265
        values for the given command, and returns an L{AmpBox} containing
4394
266
        argument values for the named command, or a L{Deferred} that fires the
4395
267
        same.
4396
268
        """
4397
269
4398
270
4399
271
4400
272
class AmpError(Exception):
4401
273
    """
4402
274
    Base class of all Amp-related exceptions.
4403
275
    """
4404
276
4405
277
4406
278
4407
279
class ProtocolSwitched(Exception):
4408
280
    """
4409
281
    Connections which have been switched to other protocols can no longer
4410
282
    accept traffic at the AMP level.  This is raised when you try to send it.
4411
283
    """
4412
284
4413
285
4414
286
4415
287
class OnlyOneTLS(AmpError):
4416
288
    """
4417
289
    This is an implementation limitation; TLS may only be started once per
4418
290
    connection.
4419
291
    """
4420
292
4421
293
4422
294
4423
295
class NoEmptyBoxes(AmpError):
4424
296
    """
4425
297
    You can't have empty boxes on the connection.  This is raised when you
4426
298
    receive or attempt to send one.
4427
299
    """
4428
300
4429
301
4430
302
4431
303
class InvalidSignature(AmpError):
4432
304
    """
4433
305
    You didn't pass all the required arguments.
4434
306
    """
4435
307
4436
308
4437
309
4438
310
class TooLong(AmpError):
4439
311
    """
4440
312
    One of the protocol's length limitations was violated.
4441
313
4442
314
    @ivar isKey: true if the string being encoded in a key position, false if
4443
315
    it was in a value position.
4444
316
4445
317
    @ivar isLocal: Was the string encoded locally, or received too long from
4446
318
    the network?  (It's only physically possible to encode "too long" values on
4447
319
    the network for keys.)
4448
320
4449
321
    @ivar value: The string that was too long.
4450
322
4451
323
    @ivar keyName: If the string being encoded was in a value position, what
4452
324
    key was it being encoded for?
4453
325
    """
4454
326
4455
327
    def __init__(self, isKey, isLocal, value, keyName=None):
4456
328
        AmpError.__init__(self)
4457
329
        self.isKey = isKey
4458
330
        self.isLocal = isLocal
4459
331
        self.value = value
4460
332
        self.keyName = keyName
4461
333
4462
334
4463
335
    def __repr__(self):
4464
336
        hdr = self.isKey and "key" or "value"
4465
337
        if not self.isKey:
4466
338
            hdr += ' ' + repr(self.keyName)
4467
339
        lcl = self.isLocal and "local" or "remote"
4468
340
        return "%s %s too long: %d" % (lcl, hdr, len(self.value))
4469
341
4470
342
4471
343
4472
344
class BadLocalReturn(AmpError):
4473
345
    """
4474
346
    A bad value was returned from a local command; we were unable to coerce it.
4475
347
    """
4476
348
    def __init__(self, message, enclosed):
4477
349
        AmpError.__init__(self)
4478
350
        self.message = message
4479
351
        self.enclosed = enclosed
4480
352
4481
353
4482
354
    def __repr__(self):
4483
355
        return self.message + " " + self.enclosed.getBriefTraceback()
4484
356
4485
357
    __str__ = __repr__
4486
358
4487
359
4488
360
4489
361
class RemoteAmpError(AmpError):
4490
362
    """
4491
363
    This error indicates that something went wrong on the remote end of the
4492
364
    connection, and the error was serialized and transmitted to you.
4493
365
    """
4494
366
    def __init__(self, errorCode, description, fatal=False, local=None):
4495
367
        """Create a remote error with an error code and description.
4496
368
4497
369
        @param errorCode: the AMP error code of this error.
4498
370
4499
371
        @param description: some text to show to the user.
4500
372
4501
373
        @param fatal: a boolean, true if this error should terminate the
4502
374
        connection.
4503
375
4504
376
        @param local: a local Failure, if one exists.
4505
377
        """
4506
378
        if local:
4507
379
            localwhat = ' (local)'
4508
380
            othertb = local.getBriefTraceback()
4509
381
        else:
4510
382
            localwhat = ''
4511
383
            othertb = ''
4512
384
        Exception.__init__(self, "Code<%s>%s: %s%s" % (
4513
385
                errorCode, localwhat,
4514
386
                description, othertb))
4515
387
        self.local = local
4516
388
        self.errorCode = errorCode
4517
389
        self.description = description
4518
390
        self.fatal = fatal
4519
391
4520
392
4521
393
4522
394
class UnknownRemoteError(RemoteAmpError):
4523
395
    """
4524
396
    This means that an error whose type we can't identify was raised from the
4525
397
    other side.
4526
398
    """
4527
399
    def __init__(self, description):
4528
400
        errorCode = UNKNOWN_ERROR_CODE
4529
401
        RemoteAmpError.__init__(self, errorCode, description)
4530
402
4531
403
4532
404
4533
405
class MalformedAmpBox(AmpError):
4534
406
    """
4535
407
    This error indicates that the wire-level protocol was malformed.
4536
408
    """
4537
409
4538
410
4539
411
4540
412
class UnhandledCommand(AmpError):
4541
413
    """
4542
414
    A command received via amp could not be dispatched.
4543
415
    """
4544
416
4545
417
4546
418
4547
419
class IncompatibleVersions(AmpError):
4548
420
    """
4549
421
    It was impossible to negotiate a compatible version of the protocol with
4550
422
    the other end of the connection.
4551
423
    """
4552
424
4553
425
4554
426
PROTOCOL_ERRORS = {UNHANDLED_ERROR_CODE: UnhandledCommand}
4555
427
4556
428
class AmpBox(dict):
4557
429
    """
4558
430
    I am a packet in the AMP protocol, much like a regular str:str dictionary.
4559
431
    """
4560
432
    __slots__ = []              # be like a regular dictionary, don't magically
4561
433
                                # acquire a __dict__...
4562
434
4563
435
4564
436
    def copy(self):
4565
437
        """
4566
438
        Return another AmpBox just like me.
4567
439
        """
4568
440
        newBox = self.__class__()
4569
441
        newBox.update(self)
4570
442
        return newBox
4571
443
4572
444
4573
445
    def serialize(self):
4574
446
        """
4575
447
        Convert me into a wire-encoded string.
4576
448
4577
449
        @return: a str encoded according to the rules described in the module
4578
450
        docstring.
4579
451
        """
4580
452
        i = self.items()
4581
453
        i.sort()
4582
454
        L = []
4583
455
        w = L.append
4584
456
        for k, v in i:
4585
457
            if len(k) > MAX_KEY_LENGTH:
4586
458
                raise TooLong(True, True, k, None)
4587
459
            if len(v) > MAX_VALUE_LENGTH:
4588
460
                raise TooLong(False, True, v, k)
4589
461
            for kv in k, v:
4590
462
                w(pack("!H", len(kv)))
4591
463
                w(kv)
4592
464
        w(pack("!H", 0))
4593
465
        return ''.join(L)
4594
466
4595
467
4596
468
    def _sendTo(self, proto):
4597
469
        """
4598
470
        Serialize and send this box to a Amp instance.  By the time it is being
4599
471
        sent, several keys are required.  I must have exactly ONE of::
4600
472
4601
473
            _ask
4602
474
            _answer
4603
475
            _error
4604
476
4605
477
        If the '_ask' key is set, then the '_command' key must also be
4606
478
        set.
4607
479
4608
480
        @param proto: an AMP instance.
4609
481
        """
4610
482
        proto.sendBox(self)
4611
483
4612
484
    def __repr__(self):
4613
485
        return 'AmpBox(%s)' % (dict.__repr__(self),)
4614
486
4615
487
# amp.Box => AmpBox
4616
488
4617
489
Box = AmpBox
4618
490
4619
491
class QuitBox(AmpBox):
4620
492
    """
4621
493
    I am an AmpBox that, upon being sent, terminates the connection.
4622
494
    """
4623
495
    __slots__ = []
4624
496
4625
497
4626
498
    def __repr__(self):
4627
499
        return 'QuitBox(**%s)' % (super(QuitBox, self).__repr__(),)
4628
500
4629
501
4630
502
    def _sendTo(self, proto):
4631
503
        """
4632
504
        Immediately call loseConnection after sending.
4633
505
        """
4634
506
        super(QuitBox, self)._sendTo(proto)
4635
507
        proto.transport.loseConnection()
4636
508
4637
509
4638
510
4639
511
class _SwitchBox(AmpBox):
4640
512
    """
4641
513
    Implementation detail of ProtocolSwitchCommand: I am a AmpBox which sets
4642
514
    up state for the protocol to switch.
4643
515
    """
4644
516
4645
517
    # DON'T set __slots__ here; we do have an attribute.
4646
518
4647
519
    def __init__(self, innerProto, **kw):
4648
520
        """
4649
521
        Create a _SwitchBox with the protocol to switch to after being sent.
4650
522
4651
523
        @param innerProto: the protocol instance to switch to.
4652
524
        @type innerProto: an IProtocol provider.
4653
525
        """
4654
526
        super(_SwitchBox, self).__init__(**kw)
4655
527
        self.innerProto = innerProto
4656
528
4657
529
4658
530
    def __repr__(self):
4659
531
        return '_SwitchBox(%r, **%s)' % (self.innerProto,
4660
532
                                         dict.__repr__(self),)
4661
533
4662
534
4663
535
    def _sendTo(self, proto):
4664
536
        """
4665
537
        Send me; I am the last box on the connection.  All further traffic will be
4666
538
        over the new protocol.
4667
539
        """
4668
540
        super(_SwitchBox, self)._sendTo(proto)
4669
541
        proto._lockForSwitch()
4670
542
        proto._switchTo(self.innerProto)
4671
543
4672
544
4673
545
4674
546
class BoxDispatcher:
4675
547
    """
4676
548
    A L{BoxDispatcher} dispatches '_ask', '_answer', and '_error' L{AmpBox}es,
4677
549
    both incoming and outgoing, to their appropriate destinations.
4678
550
4679
551
    Outgoing commands are converted into L{Deferred}s and outgoing boxes, and
4680
552
    associated tracking state to fire those L{Deferred} when '_answer' boxes
4681
553
    come back.  Incoming '_answer' and '_error' boxes are converted into
4682
554
    callbacks and errbacks on those L{Deferred}s, respectively.
4683
555
4684
556
    Incoming '_ask' boxes are converted into method calls on a supplied method
4685
557
    locator.
4686
558
4687
559
    @ivar _outstandingRequests: a dictionary mapping request IDs to
4688
560
    L{Deferred}s which were returned for those requests.
4689
561
4690
562
    @ivar locator: an object with a L{locateResponder} method that locates a
4691
563
    responder function that takes a Box and returns a result (either a Box or a
4692
564
    Deferred which fires one).
4693
565
4694
566
    @ivar boxSender: an object which can send boxes, via the L{_sendBox}
4695
567
    method, such as an L{AMP} instance.
4696
568
    @type boxSender: L{IBoxSender}
4697
569
    """
4698
570
4699
571
    implements(IBoxReceiver)
4700
572
4701
573
    _failAllReason = None
4702
574
    _outstandingRequests = None
4703
575
    _counter = 0L
4704
576
    boxSender = None
4705
577
4706
578
    def __init__(self, locator):
4707
579
        self._outstandingRequests = {}
4708
580
        self.locator = locator
4709
581
4710
582
4711
583
    def startReceivingBoxes(self, boxSender):
4712
584
        """
4713
585
        The given boxSender is going to start calling boxReceived on this
4714
586
        L{BoxDispatcher}.
4715
587
4716
588
        @param boxSender: The L{IBoxSender} to send command responses to.
4717
589
        """
4718
590
        self.boxSender = boxSender
4719
591
4720
592
4721
593
    def stopReceivingBoxes(self, reason):
4722
594
        """
4723
595
        No further boxes will be received here.  Terminate all currently
4724
596
        oustanding command deferreds with the given reason.
4725
597
        """
4726
598
        self.failAllOutgoing(reason)
4727
599
4728
600
4729
601
    def failAllOutgoing(self, reason):
4730
602
        """
4731
603
        Call the errback on all outstanding requests awaiting responses.
4732
604
4733
605
        @param reason: the Failure instance to pass to those errbacks.
4734
606
        """
4735
607
        self._failAllReason = reason
4736
608
        OR = self._outstandingRequests.items()
4737
609
        self._outstandingRequests = None # we can never send another request
4738
610
        for key, value in OR:
4739
611
            value.errback(reason)
4740
612
4741
613
4742
614
    def _nextTag(self):
4743
615
        """
4744
616
        Generate protocol-local serial numbers for _ask keys.
4745
617
4746
618
        @return: a string that has not yet been used on this connection.
4747
619
        """
4748
620
        self._counter += 1
4749
621
        return '%x' % (self._counter,)
4750
622
4751
623
4752
624
    def _sendBoxCommand(self, command, box, requiresAnswer=True):
4753
625
        """
4754
626
        Send a command across the wire with the given C{amp.Box}.
4755
627
4756
628
        Mutate the given box to give it any additional keys (_command, _ask)
4757
629
        required for the command and request/response machinery, then send it.
4758
630
4759
631
        If requiresAnswer is True, returns a C{Deferred} which fires when a
4760
632
        response is received. The C{Deferred} is fired with an C{amp.Box} on
4761
633
        success, or with an C{amp.RemoteAmpError} if an error is received.
4762
634
4763
635
        If the Deferred fails and the error is not handled by the caller of
4764
636
        this method, the failure will be logged and the connection dropped.
4765
637
4766
638
        @param command: a str, the name of the command to issue.
4767
639
4768
640
        @param box: an AmpBox with the arguments for the command.
4769
641
4770
642
        @param requiresAnswer: a boolean.  Defaults to True.  If True, return a
4771
643
        Deferred which will fire when the other side responds to this command.
4772
644
        If False, return None and do not ask the other side for acknowledgement.
4773
645
4774
646
        @return: a Deferred which fires the AmpBox that holds the response to
4775
647
        this command, or None, as specified by requiresAnswer.
4776
648
4777
649
        @raise ProtocolSwitched: if the protocol has been switched.
4778
650
        """
4779
651
        if self._failAllReason is not None:
4780
652
            return fail(self._failAllReason)
4781
653
        box[COMMAND] = command
4782
654
        tag = self._nextTag()
4783
655
        if requiresAnswer:
4784
656
            box[ASK] = tag
4785
657
        box._sendTo(self.boxSender)
4786
658
        if requiresAnswer:
4787
659
            result = self._outstandingRequests[tag] = Deferred()
4788
660
        else:
4789
661
            result = None
4790
662
        return result
4791
663
4792
664
4793
665
    def callRemoteString(self, command, requiresAnswer=True, **kw):
4794
666
        """
4795
667
        This is a low-level API, designed only for optimizing simple messages
4796
668
        for which the overhead of parsing is too great.
4797
669
4798
670
        @param command: a str naming the command.
4799
671
4800
672
        @param kw: arguments to the amp box.
4801
673
4802
674
        @param requiresAnswer: a boolean.  Defaults to True.  If True, return a
4803
675
        Deferred which will fire when the other side responds to this command.
4804
676
        If False, return None and do not ask the other side for acknowledgement.
4805
677
4806
678
        @return: a Deferred which fires the AmpBox that holds the response to
4807
679
        this command, or None, as specified by requiresAnswer.
4808
680
        """
4809
681
        box = Box(kw)
4810
682
        return self._sendBoxCommand(command, box)
4811
683
4812
684
4813
685
    def callRemote(self, commandType, *a, **kw):
4814
686
        """
4815
687
        This is the primary high-level API for sending messages via AMP.  Invoke it
4816
688
        with a command and appropriate arguments to send a message to this
4817
689
        connection's peer.
4818
690
4819
691
        @param commandType: a subclass of Command.
4820
692
        @type commandType: L{type}
4821
693
4822
694
        @param a: Positional (special) parameters taken by the command.
4823
695
        Positional parameters will typically not be sent over the wire.  The
4824
696
        only command included with AMP which uses positional parameters is
4825
697
        L{ProtocolSwitchCommand}, which takes the protocol that will be
4826
698
        switched to as its first argument.
4827
699
4828
700
        @param kw: Keyword arguments taken by the command.  These are the
4829
701
        arguments declared in the command's 'arguments' attribute.  They will
4830
702
        be encoded and sent to the peer as arguments for the L{commandType}.
4831
703
4832
704
        @return: If L{commandType} has a C{requiresAnswer} attribute set to
4833
705
        L{False}, then return L{None}.  Otherwise, return a L{Deferred} which
4834
706
        fires with a dictionary of objects representing the result of this
4835
707
        call.  Additionally, this L{Deferred} may fail with an exception
4836
708
        representing a connection failure, with L{UnknownRemoteError} if the
4837
709
        other end of the connection fails for an unknown reason, or with any
4838
710
        error specified as a key in L{commandType}'s C{errors} dictionary.
4839
711
        """
4840
712
4841
713
        # XXX this takes command subclasses and not command objects on purpose.
4842
714
        # There's really no reason to have all this back-and-forth between
4843
715
        # command objects and the protocol, and the extra object being created
4844
716
        # (the Command instance) is pointless.  Command is kind of like
4845
717
        # Interface, and should be more like it.
4846
718
4847
719
        # In other words, the fact that commandType is instantiated here is an
4848
720
        # implementation detail.  Don't rely on it.
4849
721
4850
722
        co = commandType(*a, **kw)
4851
723
        return co._doCommand(self)
4852
724
4853
725
4854
726
    def unhandledError(self, failure):
4855
727
        """
4856
728
        This is a terminal callback called after application code has had a
4857
729
        chance to quash any errors.
4858
730
        """
4859
731
        return self.boxSender.unhandledError(failure)
4860
732
4861
733
4862
734
    def _answerReceived(self, box):
4863
735
        """
4864
736
        An AMP box was received that answered a command previously sent with
4865
737
        L{callRemote}.
4866
738
4867
739
        @param box: an AmpBox with a value for its L{ANSWER} key.
4868
740
        """
4869
741
        question = self._outstandingRequests.pop(box[ANSWER])
4870
742
        question.addErrback(self.unhandledError)
4871
743
        question.callback(box)
4872
744
4873
745
4874
746
    def _errorReceived(self, box):
4875
747
        """
4876
748
        An AMP box was received that answered a command previously sent with
4877
749
        L{callRemote}, with an error.
4878
750
4879
751
        @param box: an L{AmpBox} with a value for its L{ERROR}, L{ERROR_CODE},
4880
752
        and L{ERROR_DESCRIPTION} keys.
4881
753
        """
4882
754
        question = self._outstandingRequests.pop(box[ERROR])
4883
755
        question.addErrback(self.unhandledError)
4884
756
        errorCode = box[ERROR_CODE]
4885
757
        description = box[ERROR_DESCRIPTION]
4886
758
        if errorCode in PROTOCOL_ERRORS:
4887
759
            exc = PROTOCOL_ERRORS[errorCode](errorCode, description)
4888
760
        else:
4889
761
            exc = RemoteAmpError(errorCode, description)
4890
762
        question.errback(Failure(exc))
4891
763
4892
764
4893
765
    def _commandReceived(self, box):
4894
766
        """
4895
767
        @param box: an L{AmpBox} with a value for its L{COMMAND} and L{ASK}
4896
768
        keys.
4897
769
        """
4898
770
        cmd = box[COMMAND]
4899
771
        def formatAnswer(answerBox):
4900
772
            answerBox[ANSWER] = box[ASK]
4901
773
            return answerBox
4902
774
        def formatError(error):
4903
775
            if error.check(RemoteAmpError):
4904
776
                code = error.value.errorCode
4905
777
                desc = error.value.description
4906
778
                if error.value.fatal:
4907
779
                    errorBox = QuitBox()
4908
780
                else:
4909
781
                    errorBox = AmpBox()
4910
782
            else:
4911
783
                errorBox = QuitBox()
4912
784
                log.err(error) # here is where server-side logging happens
4913
785
                               # if the error isn't handled
4914
786
                code = UNKNOWN_ERROR_CODE
4915
787
                desc = "Unknown Error"
4916
788
            errorBox[ERROR] = box[ASK]
4917
789
            errorBox[ERROR_DESCRIPTION] = desc
4918
790
            errorBox[ERROR_CODE] = code
4919
791
            return errorBox
4920
792
        deferred = self.dispatchCommand(box)
4921
793
        if ASK in box:
4922
794
            deferred.addCallbacks(formatAnswer, formatError)
4923
795
            deferred.addCallback(self._safeEmit)
4924
796
        deferred.addErrback(self.unhandledError)
4925
797
4926
798
4927
799
    def ampBoxReceived(self, box):
4928
800
        """
4929
801
        An AmpBox was received, representing a command, or an answer to a
4930
802
        previously issued command (either successful or erroneous).  Respond to
4931
803
        it according to its contents.
4932
804
4933
805
        @param box: an AmpBox
4934
806
4935
807
        @raise NoEmptyBoxes: when a box is received that does not contain an
4936
808
        '_answer', '_command' / '_ask', or '_error' key; i.e. one which does not
4937
809
        fit into the command / response protocol defined by AMP.
4938
810
        """
4939
811
        if ANSWER in box:
4940
812
            self._answerReceived(box)
4941
813
        elif ERROR in box:
4942
814
            self._errorReceived(box)
4943
815
        elif COMMAND in box:
4944
816
            self._commandReceived(box)
4945
817
        else:
4946
818
            raise NoEmptyBoxes(box)
4947
819
4948
820
4949
821
    def _safeEmit(self, aBox):
4950
822
        """
4951
823
        Emit a box, ignoring L{ProtocolSwitched} and L{ConnectionLost} errors
4952
824
        which cannot be usefully handled.
4953
825
        """
4954
826
        try:
4955
827
            aBox._sendTo(self.boxSender)
4956
828
        except (ProtocolSwitched, ConnectionLost):
4957
829
            pass
4958
830
4959
831
4960
832
    def dispatchCommand(self, box):
4961
833
        """
4962
834
        A box with a _command key was received.
4963
835
4964
836
        Dispatch it to a local handler call it.
4965
837
4966
838
        @param proto: an AMP instance.
4967
839
        @param box: an AmpBox to be dispatched.
4968
840
        """
4969
841
        cmd = box[COMMAND]
4970
842
        responder = self.locator.locateResponder(cmd)
4971
843
        if responder is None:
4972
844
            return fail(RemoteAmpError(
4973
845
                    UNHANDLED_ERROR_CODE,
4974
846
                    "Unhandled Command: %r" % (cmd,),
4975
847
                    False,
4976
848
                    local=Failure(UnhandledCommand())))
4977
849
        return maybeDeferred(responder, box)
4978
850
4979
851
4980
852
4981
853
class CommandLocator:
4982
854
    """
4983
855
    A L{CommandLocator} is a collection of responders to AMP L{Command}s, with
4984
856
    the help of the L{Command.responder} decorator.
4985
857
    """
4986
858
4987
859
    class __metaclass__(type):
4988
860
        """
4989
861
        This metaclass keeps track of all of the Command.responder-decorated
4990
862
        methods defined since the last CommandLocator subclass was defined.  It
4991
863
        assumes (usually correctly, but unfortunately not necessarily so) that
4992
864
        those commands responders were all declared as methods of the class
4993
865
        being defined.  Note that this list can be incorrect if users use the
4994
866
        Command.responder decorator outside the context of a CommandLocator
4995
867
        class declaration.
4996
868
4997
869
        The Command.responder decorator explicitly cooperates with this
4998
870
        metaclass.
4999
871
        """
5000
872
Reviewer	Date Requested	Status
Björn Tillenius (community)	2013-04-05	Approve on 2013-04-09
Alberto Donato (community)	2013-04-05	Approve on 2013-04-08
Review via email: mp+157421@code.launchpad.net