Merge lp:~cmiller/desktopcouch/pairing-fixups into lp:desktopcouch

[Chad Miller]

In pairing and replication, be smarter about the bind address of the desktopcouch daemon.

Fix a problem with records marked as deleted not excluded from the default view code. Add versioning of view functions to force upgrades of old, bad, function.

[Chad Miller (cmiller)]

Fix Bug#419969: at pairing time, change couchdb pairing address to public

Fix Bug#419973: in replication daemon, be sure local couchdb bind address is not 127/8 .

[John O'Brien (jdobrien)]

Looks good, tests run. I'm unsure how to test if this stuff works yet,

review: Approve

[Tim Cole (tcole)]

Looks reasonable. I don't really care for the JavaScript one-liner though; could it perhaps be formatted a little more legibly? (If you want the output JavaScript to be a one-liner, you still have the option of breaking the python string so that it at least appears formatted/indented reasonably in the Python source.)

review: Approve

[lp:~cmiller/desktopcouch/pairing-fixups]

54. By Chad Miller on 2009-09-01

Reformat JavaScript blob of code in server views code.

=== modified file 'bin/desktopcouch-pair'
--- bin/desktopcouch-pair	2009-08-26 18:01:29 +0000
+++ bin/desktopcouch-pair	2009-08-31 15:27:31 +0000
@@ -64,6 +64,7 @@
 
 from desktopcouch.pair.couchdb_pairing import network_io
 from desktopcouch.pair.couchdb_pairing import dbus_io
+from desktopcouch.pair import pairing_record_type
 
 discovery_tool_version = "1"
 
@@ -697,7 +698,7 @@
     # Create a paired server record
     service_data = CLOUD_SERVICES[name]
     data = {
-        "record_type": "http://www.freedesktop.org/wiki/Specifications/desktopcouch/paired_server",
+        "record_type": pairing_record_type,
         "pairing_identifier": str(uuid.uuid4()),
         "server": "%s:%s" % (hostname, port),
         "oauth": {
@@ -733,6 +734,43 @@
     success_note.run()
     success_note.destroy()
 
+
+def set_couchdb_bind_address():
+    from desktopcouch.records.server import CouchDatabase
+    from desktopcouch import local_files
+    bind_address = local_files.get_bind_address()
+
+    if bind_address not in ("127.0.0.1", "0.0.0.0", "::1", None):
+        logging.info("we're not qualified to change explicit address %s",
+                bind_address)
+        return False
+
+    db = CouchDatabase("management", create=True)
+    results = db.get_records(create_view=True)
+    count = 0
+    for row in results[pairing_record_type]:
+        if "server" in row.value and row.value["server"] != "":
+            # Is the record of something that probably connects back to us?
+            logging.debug("not counting fully-addressed machine %r", row.value["server"])
+            continue
+        count += 1
+    logging.debug("paired machine count is %d", count)
+    if count > 0:
+        if ":" in bind_address:
+            want_bind_address = "::0"
+        else:
+            want_bind_address = "0.0.0.0"
+    else:
+        if ":" in bind_address:
+            want_bind_address = "::0"
+        else:
+            want_bind_address = "127.0.0.1"
+
+    if bind_address != want_bind_address:
+        local_files.set_bind_address(want_bind_address)
+        logging.warning("changing the desktopcouch bind address from %r to %r",
+                bind_address, want_bind_address)
+
 def main(args):
     """Start execution."""
     global pick_or_listen  # pylint: disable-msg=W0601
@@ -747,6 +785,7 @@
         pick_or_listen = PickOrListen()
         return run_program()
     finally:
+        set_couchdb_bind_address()
         logging.debug("exiting couchdb pairing tool")
 
 
 
=== modified file 'bin/desktopcouch-paired-replication-manager'
--- bin/desktopcouch-paired-replication-manager	2009-08-26 19:04:39 +0000
+++ bin/desktopcouch-paired-replication-manager	2009-08-30 12:25:52 +0000
@@ -32,6 +32,7 @@
 import xdg.BaseDirectory
 
 import desktopcouch
+from desktopcouch import local_files
 from desktopcouch.pair.couchdb_pairing import couchdb_io
 from desktopcouch.pair.couchdb_pairing import dbus_io
 
@@ -84,12 +85,37 @@
     rotating_log.setLevel(logging.DEBUG)
     formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
     rotating_log.setFormatter(formatter)
+    console = logging.StreamHandler()
+    console.setLevel(logging.WARNING)
     logging.getLogger('').addHandler(rotating_log)
+    logging.getLogger('').addHandler(console)
     logging.getLogger('').setLevel(logging.DEBUG)
 
     try:
         log.info("Starting.")
 
+        bind_addr = local_files.get_bind_address()
+        if bind_addr is not None and bind_addr != '':
+            from socket import inet_ntop, inet_pton
+            from socket import error as socketerror
+            try:
+                # There are more than dotted decimal quad to
+                # contend with.  183468213 == 0xaef80b5 == 
+                # 0xa.0xef.0x80.0xb5 == 012.0357.0200.0265 ==
+                # 10.239.128.181.  Just ask the system instead.
+                addr_standard_form = inet_ntop(inet_pton(bind_addr))
+            except socketerror:
+                log.warn("bind address is illegal, %r", bind_addr)
+                log.warn("(If couchdb does understand it, please open a bug!)")
+                sys.exit(1)
+                
+            if addr_standard_form.startswith("127."):  # entire /8 is local.
+                log.warn("couchdb bound to addr %r; cannot accept connections",
+                        bind_addr)
+            elif addr_standard_form == "::1":  # no addr block on v6.  Just one.
+                log.warn("couchdb bound to addr %r; cannot accept connections",
+                        bind_addr)
+
         unique_identifiers = couchdb_io.get_my_host_unique_id()
         if unique_identifiers is None:
             log.warn("No unique hostaccount id is set, so pairing not enabled.")
 
=== modified file 'desktopcouch/local_files.py'
--- desktopcouch/local_files.py	2009-08-25 16:01:15 +0000
+++ desktopcouch/local_files.py	2009-08-31 15:27:31 +0000
@@ -26,7 +26,11 @@
 import os
 import xdg.BaseDirectory
 import subprocess
-
+import logging
+try:
+    import ConfigParser as configparser
+except ImportError:
+    import configparser
 
 def mkpath(rootdir, path):
     "Remove .. from paths"
@@ -57,7 +61,10 @@
         stdout=subprocess.PIPE)
     line = process.stdout.read().split('\n')[0]
     couchversion = line.split()[-1]
-    if couchversion.startswith('0.1'):
+
+    import distutils.version
+    if distutils.version.LooseVersion(couchversion) >= \
+            distutils.version.LooseVersion('0.10'):
         chain = '-a'
     else:
         chain = '-C'
@@ -65,10 +72,13 @@
     return chain
 
 class NoOAuthTokenException(Exception):
+    def __init__(self, file_name):
+        super(Exception, self).__init__()
+        self.file_name = file_name
     def __str__(self):
-        return "OAuth details were not found in the ini file (%s)" % FILE_INI
+        return "OAuth details were not found in the ini file (%s)" % self.file_name
 
-def get_oauth_tokens():
+def get_oauth_tokens(config_file_name=FILE_INI):
     """Return the OAuth tokens from the desktop Couch ini file.
        CouchDB OAuth is two-legged OAuth (not three-legged like most OAuth).
        We have one "consumer", defined by a consumer_key and a secret,
@@ -78,18 +88,17 @@
        (More traditional 3-legged OAuth starts with a "request token" which is
        then used to procure an "access token". We do not require this.)
     """
-    import ConfigParser
-    c = ConfigParser.ConfigParser()
+    c = configparser.ConfigParser()
     # monkeypatch ConfigParser to stop it lower-casing option names
     c.optionxform = lambda s: s
-    c.read(FILE_INI)
+    c.read(config_file_name)
     try:
         oauth_token_secrets = c.items("oauth_token_secrets")
         oauth_consumer_secrets = c.items("oauth_consumer_secrets")
-    except ConfigParser.NoSectionError:
-        raise NoOAuthTokenException
+    except configparser.NoSectionError:
+        raise NoOAuthTokenException(config_file_name)
     if not oauth_token_secrets or not oauth_consumer_secrets:
-        raise NoOAuthTokenException
+        raise NoOAuthTokenException(config_file_name)
     try:
         out = {
             "token": oauth_token_secrets[0][0],
@@ -98,9 +107,30 @@
             "consumer_secret": oauth_consumer_secrets[0][1]
         }
     except IndexError:
-        raise NoOAuthTokenException
+        raise NoOAuthTokenException(config_file_name)
     return out
 
+
+def get_bind_address(config_file_name=FILE_INI):
+    """Retreive a string if it exists, or None if it doesn't."""
+    c = configparser.ConfigParser()
+    try:
+        c.read(config_file_name)
+        return c.get("httpd", "bind_address")
+    except (configparser.NoOptionError, OSError), e:
+        logging.warn("config file %r error. %s", config_file_name, e)
+        return None
+
+def set_bind_address(address, config_file_name=FILE_INI):
+    c = configparser.SafeConfigParser()
+    c.read(config_file_name)
+    if not c.has_section("httpd"):
+        c.add_section("httpd")
+    c.set("httpd", "bind_address", address)
+    with open(config_file_name, 'wb') as configfile:
+        c.write(configfile)
+
+
 # You will need to add -b or -k on the end of this
 COUCH_EXEC_COMMAND = [COUCH_EXE, couch_chain_flag(), FILE_INI, '-p', FILE_PID,
   '-o', FILE_STDOUT, '-e', FILE_STDERR]
 
=== modified file 'desktopcouch/pair/__init__.py'
--- desktopcouch/pair/__init__.py	2009-07-08 17:48:11 +0000
+++ desktopcouch/pair/__init__.py	2009-08-31 15:27:31 +0000
@@ -14,3 +14,5 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with desktopcouch.  If not, see <http://www.gnu.org/licenses/>.
 """The pair module."""
+
+pairing_record_type = "http://www.freedesktop.org/wiki/Specifications/desktopcouch/paired_server"
 
=== modified file 'desktopcouch/records/server.py'
--- desktopcouch/records/server.py	2009-08-24 20:35:25 +0000
+++ desktopcouch/records/server.py	2009-08-31 15:28:46 +0000
@@ -44,6 +44,17 @@
                 "passing create=True)") % self.database
 
 
+def row_is_deleted(row):
+    """Test if a row is marked as deleted.  Smart views 'maps' should not
+    return rows that are marked as deleted, so this function is not often
+    required."""
+    try:
+        return row['application_annotations']['Ubuntu One']\
+                ['private_application_annotations']['deleted']
+    except KeyError:
+        return False
+
+
 class CouchDatabase(object):
     """An small records specific abstraction over a couch db database."""
 
@@ -212,10 +223,12 @@
             return []
 
     def get_records(self, record_type=None, create_view=False,
-            design_doc=DEFAULT_DESIGN_DOCUMENT):
-        """A convenience function to get records.  We optionally create a view
-        in the design document.  C<create_view> may be True or False, and a
-        special value, None, is analogous to  O_EXCL|O_CREAT .
+            design_doc=DEFAULT_DESIGN_DOCUMENT, version="1"):
+        """A convenience function to get records from a view named
+        C{get_records_and_type}, suffixed with C{__v} and the supplied version
+        string (or default of "1").  We optionally create a view in the design
+        document.  C{create_view} may be True or False, and a special value,
+        None, is analogous to  O_EXCL|O_CREAT .
 
         Set record_type to a string to retrieve records of only that
         specified type. Otherwise, usse the view to return *all* records.
@@ -233,11 +246,14 @@
         =>> people = results[['Person']:['Person','ZZZZ']]
         """
         view_name = "get_records_and_type"
-        view_map_js = """function(doc) { emit(doc.record_type, doc) }"""
+        view_map_js = """function(doc) { try  {if (! doc['application_annotations']['Ubuntu One']['private_application_annotations']['deleted']) { emit(doc.record_type, doc);} } catch (e) { emit(doc.record_type, doc); } }"""
 
         if design_doc is None:
             design_doc = view_name
 
+        if not version is None:  # versions do not affect design_doc name.
+            view_name = view_name + "__v" + version
+
         exists = self.view_exists(view_name, design_doc)
 
         if exists:
@@ -255,4 +271,3 @@
             return viewdata
         else:
             return viewdata[record_type]
-
 
=== modified file 'desktopcouch/records/tests/test_server.py'
--- desktopcouch/records/tests/test_server.py	2009-08-24 20:35:25 +0000
+++ desktopcouch/records/tests/test_server.py	2009-08-31 15:28:46 +0000
@@ -18,7 +18,7 @@
 
 """testing database/contact.py module"""
 import testtools
-from desktopcouch.records.server import CouchDatabase
+from desktopcouch.records.server import CouchDatabase, row_is_deleted
 from desktopcouch.records.record import Record
 from desktopcouch.records.tests import get_uri
 
@@ -146,10 +146,17 @@
         other_record_type = "http://example.com/unittest/bad"
 
         for i in range(7):
+            record = Record({'record_number': i},
+                    record_type=good_record_type)
             if i % 3 == 1:
                 record = Record({'record_number': i},
                         record_type=good_record_type)
                 record_ids_we_care_about.add(self.database.put_record(record))
+            elif i % 3 == 2:
+                record = Record({'record_number': i},
+                        record_type=good_record_type)
+                record_id = self.database.put_record(record)  # correct type,
+                self.database.delete_record(record_id)  # but marked deleted!
             else:
                 record = Record({'record_number': i},
                         record_type=other_record_type)
@@ -160,6 +167,7 @@
         for row in results[good_record_type]:  # index notation
             self.assertTrue(row.id in record_ids_we_care_about)
             record_ids_we_care_about.remove(row.id)
+            self.assertFalse(row_is_deleted(row))
 
         self.assertTrue(len(record_ids_we_care_about) == 0, "expected zero")