Merge lp:~mike-amy/sahana-eden/climate into lp:sahana-eden

[Mike Amy]

Reverted the error handling in the mentioned method.
Kind of scary that there are invisible rules surrounding whether or not there is a session.

==========

Fixed merge problems, thanks flavour.

views/climate/chart_popup.html uses {{include jquery.html}}
reverted file 'private/prepopulate/default/tasks.cfg'
I think that was some strange merge behaviour as I'd added that file when there was a bug around it being '.txt'.

removed the r=request from: # self.url = "%s/%s" % (URL(r=request, c="default", f="download"),

Removed single-quotes from Python code and double-quotes from javascript code. Some double quotes in the javascript code have been left in as the generated javascript data-structures are tested by parsing as JSON, which doesn't accept single-quotes.

==========

Added NetCDF importer for climate data, improved performance of the overlay layer generation.
Fixed tests, updated to use "current" global variable.

[Fran Boon (flavour)]

views/climate/chart_popup.html
Please {{include jquery.html}} instead of hardcoding the jquery version.
You are already a point version out & we'll shortly go up to the newly released 1.6.3...

This should be reverted:
=== removed file 'private/prepopulate/default/tasks.cfg'

Can remove the r=request from:
+# self.url = "%s/%s" % (URL(r=request, c="default", f="download"),

I also see a very large number of single quotes in the Python code & some double quotes in the javascript....I'm happy to clean up the odd one that slips through, but this number seems excessive for me.

review: Needs Fixing

[Fran Boon (flavour)]

Thanks for cleaning up.

self.debug isn't appropriate for use in gis.import_csv()
This function only gets used via the CLI currently.
session doesn't exist here so session.s3.debug will raise an error.
I definitely want the old behaviour - i.e. provide a warning message to the console, but don't stop what you're doing (we can choose not to db.commit() if we see the errors as fatal). No point in adding stuff to session which won't be seen.
I'm not keen on it elsewhere, but I'll accept it for now at least, but in that location it's clearly wrong.

review: Needs Fixing

=== modified file 'controllers/climate.py'
--- controllers/climate.py	2011-08-06 18:24:53 +0000
+++ controllers/climate.py	2011-09-06 11:51:25 +0000
@@ -2,92 +2,14 @@
 
 module = "climate"
 
-class ClimateDataMapPlugin(object):
-    def __init__(self,
-        data_type_option_names,
-        parameter_names,
-        projected_option_type_names,
-        year_min,
-        year_max
-    ):
-        self.data_type_option_names = data_type_option_names
-        self.parameter_names = parameter_names
-        self.projected_option_type_names = projected_option_type_names
-        self.year_min = year_min
-        self.year_max = year_max
-
-    def extend_gis_map(self, add_javascript, add_configuration):
-        add_javascript("scripts/S3/s3.gis.climate.js")
-        add_configuration(
-            SCRIPT(
-                "\n".join((
-                    "registerPlugin(",
-                    "    new ClimateDataMapPlugin("+
-                            json.dumps(
-                                dict(
-                                    self.__dict__,
-                                    data_type_label = str(T("Data Type")),
-                                    projected_option_type_label = str(T("Projection Type"))
-                                ),
-                                indent = 4
-                            )+
-                    ")",
-                    ")",
-                ))
-            )
-        )
-
-    def add_html(self, html):
-        statistics_widget = FIELDSET(
-            LEGEND("Statistics"),
-            UL(
-                _style="list-style:none",
-                *(
-                    LI(
-                        INPUT(
-                            _type="radio",
-                            _name="statistics",
-                            _id="id_%s" % statistic,
-                        ),
-                        LABEL(
-                            statistic,
-                            _for="id_%s" % statistic,
-                        )
-                    )
-                    for statistic in ["Mean", "Max", "Min"]
-                )
-            )
-        )
-
-        html.append(
-            DIV(
-                FORM(
-                    _id="controller",
-                    *(
-                        SCRIPT(
-                            _type="text/javascript",
-                            *["""
-                            """]
-                        ),
-                        climate_data_type_widget,
-                        parameters_widget,
-                        statistics_widget,
-                        period_widget
-                    )
-                )
-            )
-        )
-
-    def get_image_overlay(self, ):
-        from gluon.contenttype import contenttype
-        response.headers["Content-Type"] = contenttype(".png")
-        # @ToDo: Should be a file in static
-        return response.stream(open("/Users/mike/Desktop/red_wave.png"))
-
-climate_data_map_plugin = ClimateDataMapPlugin(
-    data_type_option_names = ["Observed", "Gridded", "Projected"],
-    parameter_names = ["Rainfall", "Temperature", "Wind", "Humidity", "Sunshine"],
-    projected_option_type_names = ["RC Model", "GC Model", "Scenario"],
+ClimateDataPortal = local_import("ClimateDataPortal")
+
+sample_type_names = ClimateDataPortal.sample_codes.keys()
+variable_names = ClimateDataPortal.tables.keys()
+
+map_plugin = ClimateDataPortal.MapPlugin(
+    data_type_option_names = sample_type_names,
+    parameter_names = variable_names,
     year_max = datetime.date.today().year,
     year_min = 1960,
 )
@@ -120,16 +42,16 @@
         print_tool = {"url": print_service}
     else:
         print_tool = {}
-
+    
     map = gis.show_map(
+        lat = 28.5,
+        lon = 84.1,
+        zoom = 7,
         toolbar = False,
-        catalogue_toolbar=catalogue_toolbar, # T/F, top tabs toolbar
+#        catalogue_toolbar=catalogue_toolbar, # T/F, top tabs toolbar
         wms_browser = wms_browser, # dict
-        catalogue_layers=catalogue_layers, # T/F
-        mouse_position = deployment_settings.get_gis_mouse_position(),
-        print_tool = print_tool,
         plugins = [
-            climate_data_map_plugin
+            map_plugin
         ]
     )
 
@@ -138,7 +60,148 @@
         module_name=module_name,
         map=map
     )
-
-def climate_image_overlay():
-    return climate_data_map_plugin.get_image_overlay()
-
+    
+month_names = dict(
+    January=1,
+    February=2,
+    March=3,
+    April=4,
+    May=5,
+    June=6,
+    July=7,
+    August=8,
+    September=9,
+    October=10,
+    November=11,
+    December=12
+)
+
+for name, number in month_names.items():
+    month_names[name[:3]] = number
+for name, number in month_names.items():
+    month_names[name.upper()] = number
+for name, number in month_names.items():
+    month_names[name.lower()] = number
+
+def convert_date(default_month):
+    def converter(year_month):
+        components = year_month.split("-")
+        year = int(components[0])
+        assert 1960 <= year, "year must be >= 1960"
+        
+        try:
+            month_value = components[1]
+        except IndexError:
+            month = default_month
+        else:
+            try:
+                month = int(month_value)
+            except TypeError:
+                month = month_names[month_value]
+        
+        assert 1 <= month <= 12, "month must be in range 1:12"
+        return datetime.date(year, month, 1)
+    return converter
+
+def one_of(options):
+    def validator(choice):
+        assert choice in options, "should be one of %s, not '%s'" % (
+            options,
+            choice
+        )
+        return choice
+    return validator
+
+def climate_overlay_data():
+    kwargs = dict(request.vars)
+    kwargs["parameter"] = kwargs["parameter"].replace("+", " ")
+    
+    arguments = {}
+    errors = []
+    for kwarg_name, converter in dict(
+        data_type = one_of(sample_type_names),
+        statistic = one_of(("Maximum", "Minimum", "Average")),
+        parameter = one_of(variable_names),
+        from_date = convert_date(default_month = 1),
+        to_date = convert_date(default_month = 12),
+    ).iteritems():
+        try:
+            value = kwargs.pop(kwarg_name)
+        except KeyError:
+            errors.append("%s missing" % kwarg_name)
+        else:
+            try:
+                arguments[kwarg_name] = converter(value)
+            except TypeError:
+                errors.append("%s is wrong type" % kwarg_name)
+            except AssertionError, assertion_error:
+                errors.append("%s: %s" % (kwarg_name, assertion_error))                
+    if kwargs:
+        errors.append("Unexpected arguments: %s" % kwargs.keys())
+    
+    if errors:
+        raise HTTP(500, "<br />".join(errors))
+    else:
+        import gluon.contenttype
+        data_path = map_plugin.get_overlay_data(
+            env = Storage(globals()),
+            **arguments
+        )
+        return response.stream(
+            open(data_path,"rb"),
+            chunk_size=4096
+        )
+
+def list_of(converter):
+    def convert_list(choices):
+        return map(converter, choices)
+    return convert_list
+
+def climate_chart():
+    kwargs = dict(request.vars)
+    import simplejson as JSON
+    specs = JSON.loads(kwargs.pop("spec"))
+    
+    checked_specs = []
+    for spec in specs:
+        arguments = {}
+        errors = []
+        for name, converter in dict(
+            data_type = one_of(sample_type_names),
+            parameter = one_of(variable_names),
+            from_date = convert_date(default_month = 1),
+            to_date = convert_date(default_month = 12),
+            place_ids = list_of(int)
+        ).iteritems():
+            try:
+                value = spec.pop(name)
+            except KeyError:
+                errors.append("%s missing" % name)
+            else:
+                try:
+                    arguments[name] = converter(value)
+                except TypeError:
+                    errors.append("%s is wrong type" % name)
+                except AssertionError, assertion_error:
+                    errors.append("%s: %s" % (name, assertion_error))
+        if spec:
+            errors.append("Unexpected arguments: %s" % spec.keys())
+        checked_specs.append(arguments)
+    
+    if errors:
+        raise HTTP(500, "<br />".join(errors))
+    else:
+        import gluon.contenttype
+        response.headers["Content-Type"] = gluon.contenttype.contenttype(".png")
+        data_image_file_path = map_plugin.render_plots(
+            env = Storage(globals()),
+            specs = checked_specs
+        )
+        return response.stream(
+            open(data_image_file_path,"rb"),
+            chunk_size=4096
+        )
+        
+def chart_popup():
+    return {}
+    

=== modified file 'deployment-templates/models/000_config.py'
--- deployment-templates/models/000_config.py	2011-08-25 09:17:02 +0000
+++ deployment-templates/models/000_config.py	2011-09-06 11:51:25 +0000
@@ -249,7 +249,7 @@
     strict_hierarchy = False,
     # Should all specific locations (e.g. addresses, waypoints) be required to
     # link to where they are in the location hierarchy?
-    location_parent_required = False,
+    location_parent_required = False
 )
 # Set this if there will be multiple areas in which work is being done,
 # and a menu to select among them is wanted. With this on, any map

=== modified file 'models/03_gis.py'
--- models/03_gis.py	2011-09-05 22:10:34 +0000
+++ models/03_gis.py	2011-09-06 11:51:25 +0000
@@ -1369,6 +1369,8 @@
 # =============================================================================
 def gis_map_tables():
     """ Load the GIS Map Tables when needed """
+    if "gis_layer_bing" in db.tables:
+        return
 
     # -------------------------------------------------------------------------
     # GPS Waypoints

=== added file 'models/climate.py'
--- models/climate.py	1970-01-01 00:00:00 +0000
+++ models/climate.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+module = "climate"
+if deployment_settings.has_module(module):
+    local_import("ClimateDataPortal").define_models(env = Storage(globals()))

=== added directory 'modules/ClimateDataPortal'
=== added file 'modules/ClimateDataPortal/MapPlugin.py'
--- modules/ClimateDataPortal/MapPlugin.py	1970-01-01 00:00:00 +0000
+++ modules/ClimateDataPortal/MapPlugin.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,430 @@
+
+# notes:
+
+# dependencies:
+# R
+
+# create folder for cache:
+# mkdir -p /tmp/climate_data_portal/images/recent/
+# mkdir -p /tmp/climate_data_portal/images/older/
+
+MAX_CACHE_FOLDER_SIZE = 2**24 # 16 MiB
+
+class TwoStageCache(object):
+    def __init__(self, folder, max_size):
+        self.folder = folder
+        self.max_size
+    
+    def purge(self):
+        pass
+    
+    def retrieve(self, file_name, generate_if_not_found):
+        pass
+
+import os, errno
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc: # Python >2.5
+        if exc.errno == errno.EEXIST:
+            pass
+        else: raise
+
+def define(env, place, tables, date_to_month_number, sample_codes, exports):
+    # This starts an R interpreter.
+    # As we are sharing it (restarting it every time is inefficient), 
+    # we have to be somewhat careful to make sure objects are garbage collected
+    # better to just not stick anything in R's globals 
+    try:
+        import rpy2.robjects as robjects
+    except ImportError:
+        import logging
+        logging.getLogger().error(
+"""R is required by the climate data portal to generate charts
+
+To install R: refer to:
+http://cran.r-project.org/doc/manuals/R-admin.html
+
+
+rpy2 is required to interact with python.
+
+To install rpy2, refer to:
+http://rpy.sourceforge.net/rpy2/doc-dev/html/overview.html
+""")
+        raise
+
+    R = robjects.r
+    
+    from rpy2.robjects.packages import importr
+
+    base = importr("base")
+    
+    from math import fsum
+    def average(values):
+        "Safe float average"
+        l = len(values)
+        if l is 0:
+            return None
+        else:
+            return fsum(values)/l
+    
+    class Maximum(object):
+        def __init__(self, column, add_query_term):
+            self.value_max = value_max = column.max()
+            add_query_term(value_max)
+            
+        def __call__(self, row):
+            return row._extra[self.value_max]
+
+    class Minimum(object):
+        def __init__(self, column, add_query_term):
+            self.value_min = value_min = column.min()
+            add_query_term(value_min)
+            
+        def __call__(self, row):
+            return row._extra[self.value_min]    
+
+    class Average(object):
+        def __init__(self, column, add_query_term):
+            self.value_sum = value_sum = column.sum()
+            self.value_count = value_count = column.count()
+            add_query_term((
+                value_sum,
+                value_count
+            ))
+            
+        def __call__(self, row):
+            return row._extra[self.value_sum] / row._extra[self.value_count]
+
+    aggregators = {
+        "Maximum": Maximum,
+        "Minimum": Minimum,
+        "Average": Average
+    }
+
+    def get_cached_or_generated_file(cache_file_name, generate):
+        from os.path import join, exists
+        from os import stat, makedirs
+        # this needs to become a setting
+        climate_data_image_cache_path = join(
+            "/tmp","climate_data_portal","images"
+        )
+        recent_cache = join(climate_data_image_cache_path, "recent")
+        mkdir_p(recent_cache)
+        older_cache = join(climate_data_image_cache_path, "older")
+        mkdir_p(older_cache)
+        recent_cache_path = join(recent_cache, cache_file_name)
+        if not exists(recent_cache_path):
+            older_cache_path = join(older_cache, cache_file_name)
+            if exists(older_cache_path):
+                # move the older cache to the recent folder
+                rename(older_cache_path, recent_cache_path)
+            else:
+                generate(recent_cache_path)
+            file_path = recent_cache_path
+                
+            # update the folder size file (race condition?)
+            folder_size_file_path = join(climate_data_image_cache_path, "size")
+            folder_size_file = open(folder_size_file_path, "w+")
+            folder_size_file_contents = folder_size_file.read()
+            try:
+                folder_size = int(folder_size_file_contents)
+            except ValueError:
+                folder_size = 0
+            folder_size_file.seek(0)
+            folder_size_file.truncate()
+            folder_size += stat(file_path).st_size
+            if folder_size > MAX_CACHE_FOLDER_SIZE:
+                rmdir(older_cache)
+
+            folder_size_file.write(str(folder_size))
+            folder_size_file.close()
+        else:
+            # use the existing cached image
+            file_path = recent_cache_path
+        return file_path
+
+    class MapPlugin(object):
+        def __init__(
+            self,
+            data_type_option_names,
+            parameter_names,
+            year_min,
+            year_max
+        ):
+            self.data_type_option_names = data_type_option_names
+            self.parameter_names = parameter_names
+            self.year_min = year_min 
+            self.year_max = year_max
+
+        def extend_gis_map(self, add_javascript, add_configuration):
+            add_javascript("scripts/S3/s3.gis.climate.js")
+            SCRIPT = env.SCRIPT
+            T = env.T
+            import json
+            
+            add_configuration(
+                SCRIPT(
+                    "\n".join((
+                        "registerPlugin(",
+                        "    new ClimateDataMapPlugin("+
+                                json.dumps(
+                                    dict(
+                                        data_type_option_names = self.data_type_option_names,
+                                        parameter_names = self.parameter_names,
+                                        year_min = self.year_min,
+                                        year_max = self.year_max,
+                                        overlay_data_URL = "/%s/climate/climate_overlay_data" % (
+                                            env.request.application
+                                        ),
+                                        chart_URL = "/%s/climate/climate_chart" % (
+                                            env.request.application
+                                        ),
+                                        data_type_label = str(T("Data Type")),
+                                        projected_option_type_label = str(
+                                            T("Projection Type")
+                                        )
+                                    ),
+                                    indent = 4
+                                )+
+                            ")",
+                        ")",
+                    ))
+                )
+            )
+            
+                
+        def get_overlay_data(
+            self,
+            env,
+            data_type,
+            parameter,
+            from_date,
+            to_date,
+            statistic
+        ):            
+            from_month = date_to_month_number(from_date)
+            to_month = date_to_month_number(to_date)
+            def generate_map_overlay_data(file_path):
+                # generate the new file in the recent folder
+                
+                db = env.db
+                sample_table_name, sample_table = tables[parameter]
+                place = db.place
+                #sample_table = db[sample_table_name]
+                
+                query = [
+                    place.id,
+                    place.longitude,
+                    place.latitude,
+                ]
+                aggregator = aggregators[statistic](
+                    sample_table.value,
+                    query.append
+                )
+                
+                sample_rows = db(
+                    (sample_table.time_period >= from_month) &
+                    (sample_table.time_period <= to_month) & 
+                    (sample_table.sample_type == sample_codes[data_type]) & 
+                    (place.id == sample_table.place_id)                
+                ).select(
+                    *query,
+                    groupby=sample_table.place_id
+                )
+                
+                # map positions to data
+                # find max and min value
+                positions = {}
+                aggregated_values = []
+                for row in sample_rows:
+                    place = row.place
+                    aggregated_value = aggregator(row)
+                    aggregated_values.append(aggregated_value)
+                    positions[place.id] = (
+                        place.latitude,
+                        place.longitude,
+                        aggregated_value
+                    ) 
+                max_aggregated_value = max(aggregated_values)
+                min_aggregated_value = min(aggregated_values)
+                aggregated_range = max_aggregated_value - min_aggregated_value
+                
+                data_lines = []
+                write = data_lines.append
+                from colorsys import hsv_to_rgb
+                for id, (lat, lon, aggregated_value) in positions.iteritems():
+                    north = lat + 0.05
+                    south = lat - 0.05
+                    east = lon + 0.05
+                    west = lon - 0.05
+                    # only hue changes
+                    # hue range is from 2/3 (blue, low) to 0 (red, high) 
+                    normalised_value = 1.0-((aggregated_value - min_aggregated_value) / aggregated_range)
+                    r,g,b = hsv_to_rgb(normalised_value *(2.0/3.0), 1.0, 1.0)
+                    hex_colour = "%02x%02x%02x" % (r*255, g*255, b*255)
+                    write(
+                        "Vector("
+                            "Polygon(["
+                                "LinearRing(["
+                                    "Point(%(north)f,%(west)f),"
+                                    "Point(%(north)f,%(east)f),"
+                                    "Point(%(south)f,%(east)f),"
+                                    "Point(%(south)f,%(west)f)"
+                                "])"
+                            "]),"
+                            "{"
+                                "value:%(aggregated_value)f,"
+                                "id:%(id)i"
+                            "},"
+                            "{"
+                                "fillColor:'#%(hex_colour)s'"
+                            "}"
+                        ")," % locals()
+                    )
+                overlay_data_file = open(file_path, "w")
+                write = overlay_data_file.write
+                write("{")
+                if max_aggregated_value < 10:
+                    float_format = "%0.2f"
+                if max_aggregated_value < 100:
+                    float_format = "%0.1f"
+                elif max_aggregated_value < 10000:
+                    float_format = "%0.0f"
+                else:
+                    float_format = "%0.2e"
+                write("max:%s," % float_format % max_aggregated_value)
+                write("min:%s," % float_format % min_aggregated_value)
+                write("features:[")
+                write("".join(data_lines))
+                overlay_data_file.seek(-1, 1) # delete last ",'
+                write("]}")
+                overlay_data_file.close()
+                
+            return get_cached_or_generated_file(
+                "_".join((
+                    statistic,
+                    data_type,
+                    parameter,
+                    str(from_month),
+                    str(to_month),
+                    ".js"
+                )),
+                generate_map_overlay_data
+            )
+        
+        def render_plots(
+            self,
+            env,
+            specs
+        ):
+            def generate_chart(file_path):
+                def render_plot(
+                    data_type,
+                    parameter,
+                    from_date,
+                    to_date,
+                    place_ids
+                ):
+                    from_month = date_to_month_number(from_date)
+                    to_month = date_to_month_number(to_date)
+                    
+                    db = env.db
+                    sample_table_name, sample_table = tables[parameter]
+                    place = db.place
+                    #sample_table = db[sample_table_name]
+                    sample_rows = db(
+                        (sample_table.time_period >= from_month) &
+                        (sample_table.time_period <= to_month) & 
+                        (sample_table.sample_type == sample_codes[data_type]) & 
+                        (sample_table.place_id.belongs(place_ids))
+                    ).select(
+                        sample_table.value,
+                        sample_table.time_period,
+                    )
+                    
+                    # coalesce values by time_period:
+                    aggregated_values = {}
+                    for sample_row in sample_rows:
+                        time_period = sample_row.time_period
+                        value = sample_row.value
+                        try:
+                            aggregated_values[time_period]
+                        except KeyError:
+                            aggregated_values[time_period] = value
+                        else:
+                            aggregated_values[time_period] += value
+                    
+                    values = []
+                    time_periods = aggregated_values.keys()
+                    time_periods.sort() 
+                    for time_period in time_periods:
+                        values.append(aggregated_values[time_period])                    
+                    return from_date, to_date, data_type, parameter, values
+
+                time_serieses = []
+                c = R("c")
+                for spec in specs:
+                    from_date, to_date, data_type, parameter, values = render_plot(**spec)
+                    time_serieses.append(
+                        R("ts")(
+                            robjects.FloatVector(values),
+                            start = c(from_date.year, from_date.month),
+                            end = c(to_date.year, to_date.month),
+                            frequency = 12
+                        )
+                    )
+                
+                R("png(filename = '%s', width=640, height=480)" % file_path)
+                plot_chart = R(
+                    "function (xlab, ylab, n, ...) {"
+                        "ts.plot(...,"
+                            "gpars=list(xlab=xlab, ylab=ylab, col=c(1:n))"
+                        ")"
+                    "}"
+                )
+
+                plot_chart(
+                    "Date",
+                    "Combined %s %s" % (data_type, parameter),
+                    len(time_serieses),
+                    *time_serieses
+                )
+                R("dev.off()")
+
+            import md5
+            import gluon.contrib.simplejson as JSON
+
+            import datetime
+            def serialiseDate(obj):
+                if isinstance(
+                    obj,
+                    (
+                        datetime.date, 
+                        datetime.datetime, 
+                        datetime.time
+                    )
+                ): 
+                    return obj.isoformat()[:19].replace("T"," ") 
+                raise TypeError("%r is not JSON serializable" % (obj,)) 
+            
+            return get_cached_or_generated_file(
+                "_".join((
+                    md5.md5(
+                        JSON.dumps(
+                            specs,
+                            sort_keys=True,
+                            default=serialiseDate
+                        )
+                    ).hexdigest(),
+                    ".png"
+                )),
+                generate_chart
+            )
+            
+    exports.update(
+        MapPlugin = MapPlugin
+    )
+    
+    del globals()["define"]

=== added file 'modules/ClimateDataPortal/__init__.py'
--- modules/ClimateDataPortal/__init__.py	1970-01-01 00:00:00 +0000
+++ modules/ClimateDataPortal/__init__.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,201 @@
+
+"""
+    Climate Data Module
+
+    @author: Mike Amy
+"""
+
+# datasets are stored in actual tables
+#   - e.g. rainfall_mm
+
+# data collection points in dataset
+# values at a point within a time range
+
+# e.g. observed temperature in Kathmandu between Feb 2006 - April 2007
+
+
+sample_types = dict(
+    O = "Observed",
+    G = "Gridded",
+    
+    r = "Projected (RC)",
+    g = "Projected (GC)",
+    s = "Scenario",
+)
+
+sample_codes = {}
+
+import re
+for code, name in sample_types.iteritems():
+    globals()[re.sub("\W", "", name)] = code
+    sample_codes[name] = code
+
+
+# Until I figure out how to sanely import things from web2py,
+# apply a prophylactic import method...
+def define_models(env):
+    """
+        Define Climate Data models.
+    """    
+    db = env.db
+    Field = env.Field
+
+    def create_index(table_name, field_name):
+        db.executesql(
+            """
+            CREATE INDEX IF NOT EXISTS 
+            "index_%(table_name)s__%(field_name)s" 
+            ON "%(table_name)s" ("%(field_name)s");
+            """ % locals()
+        )
+
+    place = db.define_table(
+        "place",
+        Field(
+            "longitude",
+            "double",
+            notnull=True,
+            required=True,
+        ),
+        Field(
+            "latitude",
+            "double",
+            notnull=True,
+            required=True,
+        )
+    )
+
+    # not all places are stations with elevations
+    # as in the case of "gridded" data
+    # a station can only be in one place
+    observation_station = db.define_table(
+        "observation_station",
+        Field(
+            "id",
+            "id", # must be a place,
+            notnull=True,
+            required=True,
+        ),
+        Field(
+            "name",
+            "string",
+            notnull=True,
+            unique=False,
+            required=True,
+        ),
+        Field(
+            "elevation_metres",
+            "integer"
+        )
+    )
+
+    def sample_table(name, value_type):
+        table = db.define_table(
+            name,
+            Field(
+                "sample_type",
+                "string",
+                length = 1,
+                notnull=True,
+                # necessary as web2py requires a default value even for 
+                # not null fields                
+                default="-1", 
+                required=True
+            ),
+            Field(
+                "time_period",
+                "integer",
+                notnull=True,
+                default=-1000,
+                required=True
+            ),
+            Field(
+                # this should become a GIS field
+                "place_id",
+                place,
+                notnull=True,
+                required=True
+            ),
+            Field(
+                "value",
+                value_type,
+                notnull = True,
+                required=True,
+            ),
+        )
+        
+        create_index(name, "id")
+        create_index(name, "sample_type")
+        create_index(name, "time_period")
+        create_index(name, "place_id")
+
+        return table
+
+    rainfall_mm = sample_table("climate_rainfall_mm", "double")
+    min_temperature_celsius = sample_table("climate_min_temperature_celsius", "double")
+    max_temperature_celsius = sample_table("climate_max_temperature_celsius", "double")
+    
+    tables = {
+        "Rainfall mm": ("climate_rainfall_mm", rainfall_mm),
+        "Max Temperature C": ("climate_max_temperature_celsius", max_temperature_celsius),
+        "Min Temperature C": ("climate_min_temperature_celsius", min_temperature_celsius),
+    }
+    
+    def year_month_to_month_number(year, month):
+        """Time periods are integers representing months in years, 
+        from 1960 onwards.
+        
+        e.g. 0 = Jan 1960, 1 = Feb 1960, 12 = Jan 1961
+        
+        This function converts a year and month to a month number.
+        """
+        return ((year-1960) * 12) + (month-1)
+    
+    def date_to_month_number(date):
+        """This function converts a date to a month number.
+        
+        See also year_month_to_month_number(year, month)
+        """
+        return year_month_to_month_number(date.year, date.month)
+    
+#    def month_number_to_date(month_number):
+#        ret
+    
+    from .MapPlugin import define 
+    define(
+        env,
+        place,
+        tables,
+        date_to_month_number,
+        sample_codes,
+        globals()
+    )
+
+    # exports:
+    globals().update(
+        sample_types = sample_types,
+        
+        place = place,
+        observation_station = observation_station,
+        
+        tables = tables,
+        
+        rainfall_mm = rainfall_mm,
+        max_temperature_celsius = max_temperature_celsius,
+        min_temperature_celsius = min_temperature_celsius,
+    
+        date_to_month_number = date_to_month_number,
+        year_month_to_month_number = year_month_to_month_number,
+    )
+    
+    def redefine_models(env):
+        # avoid risking insidious aliasing bugs
+        # by not defining things more than once 
+        env.db.update(
+            climate_rainfall_mm = rainfall_mm,
+            climate_max_temperature_celsius = max_temperature_celsius,
+            climate_min_temperature_celsius = min_temperature_celsius,
+            place = place,
+            observation_station = observation_station,
+        )
+    globals()["define_models"] = redefine_models

=== added file 'modules/ClimateDataPortal/import_NetCDF_readings.py'
--- modules/ClimateDataPortal/import_NetCDF_readings.py	1970-01-01 00:00:00 +0000
+++ modules/ClimateDataPortal/import_NetCDF_readings.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,131 @@
+
+ClimateDataPortal = local_import("ClimateDataPortal")
+
+
+def get_or_create(dict, key, creator):
+    try:
+        value = dict[key]
+    except KeyError:
+        value = dict[key] = creator()
+    return value
+
+def get_or_create_record(table, query):
+    query_terms = []
+    for key, value in query.iteritems():
+        query_terms.append(getattr(table, key) == value)
+    reduced_query = reduce(
+        (lambda left, right: left & right),
+        query_terms
+    )
+    records = db(reduced_query).select()
+    count = len(records)
+    assert count <= 1, "Multiple records for %s" % query
+    if count == 0:
+        record = table.insert(**query)
+        db.commit()
+    else:
+        record = records.first()
+    return record.id
+
+def nearly(expected_float, actual_float):
+    return (expected_float * 0.999) < actual_float < (expected_float * 1.001)
+
+def add_reading_if_none(
+    database_table,
+    sample_type,
+    time_period,
+    place_id,
+    value
+):
+    records = db(
+        (database_table.sample_type == sample_type) &
+        (database_table.time_period == time_period) &
+        (database_table.place_id == place_id)
+    ).select(database_table.value, database_table.id)
+    count = len(records)
+    assert count <= 1
+    if count == 0:
+        database_table.insert(
+            sample_type = sample_type,
+            time_period = time_period,
+            place_id = place_id,
+            value = value
+        )
+    else:
+        existing = records.first()
+        assert nearly(existing.value, value), (existing.value, value, place_id)
+
+
+
+import datetime
+
+def import_climate_readings(
+    netcdf_file,
+    database_table,
+    add_reading,
+    start_time = datetime.date(1971,1,1),
+    is_undefined = lambda x: -99.900003 < x < -99.9
+):
+    """
+    Assumptions:
+        * there are no places
+        * the data is in order of places
+    """
+    variables = netcdf_file.variables
+    
+    # create grid of places
+    place_ids = {}
+    
+    def to_list(variable):
+        result = []
+        for i in range(len(variable)):
+            result.append(variable[i])
+        return result
+    
+    def iter_pairs(list):
+        for index in range(len(list)):
+            yield index, list[index]
+    
+    times = to_list(variables["time"])
+    lat = to_list(variables["lat"])
+    lon = to_list(variables["lon"])
+    for latitude in lat:
+        for longitude in lon:
+            record = get_or_create_record(
+                ClimateDataPortal.place,
+                dict(
+                    longitude = longitude,
+                    latitude = latitude
+                )
+            )
+            place_ids[(latitude, longitude)] = record
+            #print longitude, latitude, record
+    
+    tt = variables["tt"]
+    print "up to:", len(times)
+    for time_index, time in iter_pairs(times):
+        print time_index
+        time_period = start_time+datetime.timedelta(hours=time)
+        for latitude_index, latitude in iter_pairs(lat):
+            for longitude_index, longitude in iter_pairs(lon):
+                value = tt[time_index][latitude_index][longitude_index]
+                if not is_undefined(value):
+                    add_reading(
+                        database_table = database_table,
+                        sample_type = ClimateDataPortal.Gridded,
+                        time_period = ClimateDataPortal.date_to_month_number(time_period),
+                        place_id = place_ids[(latitude, longitude)],
+                        value = value
+                    )
+    db.commit()
+
+import sys
+
+from Scientific.IO import NetCDF
+
+file_name = sys.argv[1]
+import_climate_readings(
+    NetCDF.NetCDFFile(file_name),
+    ClimateDataPortal.min_temperature_celsius,
+    add_reading_if_none
+)

=== added file 'modules/ClimateDataPortal/import_stations.py'
--- modules/ClimateDataPortal/import_stations.py	1970-01-01 00:00:00 +0000
+++ modules/ClimateDataPortal/import_stations.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,53 @@
+
+ClimateDataPortal = local_import("ClimateDataPortal")
+
+from decimal import Decimal
+
+def import_stations(file_name):
+    """
+    Expects a file containing lines of the form e.g.:
+226     JALESORE             1122       172     26.65     85.78
+275     PHIDIM (PANCHTH      1419      1205     27.15     87.75
+unused  Station name         <-id    <-elev     <-lat     <-lon
+0123456789012345678901234567890123456789012345678901234567890123456789
+0         1         2         3         4         5         6
+    """
+    place = ClimateDataPortal.place
+    observation_station = ClimateDataPortal.observation_station
+    observation_station.truncate()
+    place.truncate()
+    db.commit()
+
+    for line in open(file_name, "r").readlines():
+        try:
+            place_id_text = line[27:33]
+        except IndexError:
+            continue
+        else:            
+            try:
+                place_id = int(place_id_text)
+            except ValueError:
+                continue
+            else:
+                station_name = line[8:25].strip() # don't restrict if they add more
+                elevation_metres = int(line[37:43])
+                
+                latitude = Decimal(line[47:53])
+                longitude = Decimal(line[57:623])
+                
+                assert place.insert(
+                    id  = place_id,
+                    longitude = longitude,
+                    latitude = latitude
+                ) == place_id
+                
+                station_id = observation_station.insert(
+                    id = place_id,
+                    name = station_name,
+                    elevation_metres = elevation_metres
+                )
+                print place_id, station_name, latitude, longitude, elevation_metres 
+    db.commit()
+
+import sys
+import_stations(sys.argv[1])

=== added file 'modules/ClimateDataPortal/import_tabbed_readings.py'
--- modules/ClimateDataPortal/import_tabbed_readings.py	1970-01-01 00:00:00 +0000
+++ modules/ClimateDataPortal/import_tabbed_readings.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,154 @@
+
+ClimateDataPortal = local_import("ClimateDataPortal")
+
+from decimal import Decimal
+
+
+def get_or_create(dict, key, creator):
+    try:
+        value = dict[key]
+    except KeyError:
+        value = dict[key] = creator()
+    return value
+
+import os
+
+class Readings(object):
+    def __init__(
+        self,
+        database_table,
+        null_value,
+        maximum = None,
+        minimum = None
+    ):
+        self.database_table = database_table
+        db(database_table.sample_type == ClimateDataPortal.Observed).delete()
+        self.null_value = null_value
+        self.maximum = maximum
+        self.minimum = minimum
+        
+        self.aggregated_values = {}
+     
+    def add_reading(self, time_period, reading, out_of_range):
+        if reading != self.null_value:
+            if (
+                (self.minimum is not None and reading < self.minimum) or
+                (self.maximum is not None and reading > self.maximum)
+            ):
+                out_of_range(reading)
+            else:
+                readings = get_or_create(
+                    self.aggregated_values,
+                    time_period,
+                    list
+                )
+                readings.append(reading)
+
+    def done(self, place_id):
+        for month_number, values in self.aggregated_values.iteritems():
+            self.database_table.insert(
+                sample_type = ClimateDataPortal.Observed,
+                time_period = month_number,
+                place_id = place_id,
+                value = sum(values) / len(values)
+            )
+
+import datetime
+
+
+def import_tabbed_readings(
+    folder_name,
+    variables = [],
+    place_ids = None
+):
+    """
+    Expects a folder containing files with name rtXXXX.txt
+    
+    each file contains lines of the form e.g.:
+1978\t1\t1\t0\t-99.9\t-99.9
+
+representing year, month, day, rainfall(mm), minimum and maximum temperature
+    """    
+    observation_station = ClimateDataPortal.observation_station
+    
+    null_value = Decimal("-99.9") # seems to be
+    
+    for row in db(observation_station).select(observation_station.id):
+        place_id = row.id
+        if place_ids is not None:
+            # avoid certain place ids (to allow importing particular places) 
+            start_place, end_place = map(int, place_ids.split(":"))
+            assert start_place <= end_place
+            if place_id < start_place or place_id > end_place:
+                continue
+        print place_id
+
+        data_file_path = os.path.join(folder_name, "rt%04i.txt" % place_id)
+        if not os.path.exists(data_file_path):
+            print "%s not found" % data_file_path
+        else:
+            try:
+                for line in open(data_file_path, "r").readlines():
+                    if line:
+                        data = line.split()
+                        if data:
+                            try:
+                                year = int(data[0])
+                                month = int(data[1])
+                                day = int(data[2])
+                                
+                                time_period = ClimateDataPortal.year_month_to_month_number(year, month)
+                                
+                                for variable, reading_data in zip(
+                                    variables,
+                                    data[3:6]
+                                ):
+                                    def out_of_range(reading):
+                                        print "%s/%s/%s: %s out of range" % (
+                                            day, month, year, reading
+                                        )
+                                    reading = Decimal(reading_data)
+                                    variable.add_reading(
+                                        time_period,
+                                        reading,
+                                        out_of_range = out_of_range 
+                                    )
+                                
+                            except Exception, exception:
+                                print exception
+                for variable in variables:
+                    variable.done(place_id)
+            except:
+                print line
+                raise
+                
+        db.commit()
+    else:
+        print "No stations!"
+
+import sys
+    
+null_value = Decimal("-99.9")
+import_tabbed_readings(
+    folder_name = sys.argv[1],
+    variables = [
+        Readings(
+            ClimateDataPortal.rainfall_mm,
+            null_value = null_value,
+            minimum = 0,
+        ),
+        Readings(
+            database_table = ClimateDataPortal.min_temperature_celsius,
+            null_value = null_value,
+            minimum = -120,
+            maximum = 55
+        ),
+        Readings(
+            database_table = ClimateDataPortal.max_temperature_celsius,
+            null_value = null_value,
+            minimum = -120,
+            maximum = 55
+        ),
+    ],
+    place_ids = sys.argv[2:] or None
+)

=== modified file 'modules/s3/s3gis.py'
--- modules/s3/s3gis.py	2011-09-05 22:18:45 +0000
+++ modules/s3/s3gis.py	2011-09-06 11:51:25 +0000
@@ -74,16 +74,11 @@
         Provide an easy, safe, systematic way of handling Debug output
         (print to stdout doesn't work with WSGI deployments)
     """
-    try:
-        output = "S3 Debug: %s" % str(message)
-        if value:
-            output += ": %s" % str(value)
-    except:
-        output = "S3 Debug: %s" % unicode(message)
-        if value:
-            output += ": %s" % unicode(value)
-
-    print >> sys.stderr, output
+    # should be using python's built-in logging module
+    output = u"S3 Debug: %s" % unicode(message)
+    if value:
+        output += u": %s" % unicode(value)
+    sys.stderr.write(output+"\n")
 
 SHAPELY = False
 try:
@@ -240,7 +235,6 @@
     """
 
     def __init__(self):
-
         self.deployment_settings = current.deployment_settings
         self.public_url = current.deployment_settings.get_base_public_url()
         if not current.db is not None:
@@ -308,6 +302,18 @@
         else:
             return wkt
 
+    def debug(self, message, value=None):
+        # should be using python's built-in logging module
+        session = current.session
+        if session.s3.debug:
+            raise Exception(message)
+        else:
+            output = u"S3 Debug: %s" % unicode(message)
+            if value:
+                output += u": %s" % unicode(value)
+            sys.stderr.write(output+"\n")
+            session.error = current.T(message)
+
     # -------------------------------------------------------------------------
     def download_kml(self, record_id, filename):
         """
@@ -326,11 +332,11 @@
         db = current.db
 
         layer = KMLLayer(self)
+        
         query = (layer.table.id == record_id)
         record = db(query).select(limitby=(0, 1)).first()
         url = record.url
 
-        layer.add_record(record)
         cachepath = layer.cachepath
         filepath = os.path.join(cachepath, filename)
 
@@ -427,7 +433,7 @@
                 myfile = zipfile.ZipFile(fp)
                 try:
                     file = myfile.read("doc.kml")
-                except:
+                except: # Naked except!!
                     file = myfile.read(myfile.infolist()[0].filename)
                 myfile.close()
 
@@ -531,7 +537,7 @@
             try:
                 lon = features[0].lon
                 simple = True
-            except:
+            except AttributeError:
                 simple = False
 
             for feature in features:
@@ -544,7 +550,7 @@
                         # A Join
                         lon = feature.gis_location.lon
                         lat = feature.gis_location.lat
-                except:
+                except AttributeError:
                     # Skip any rows without the necessary lat/lon fields
                     continue
 
@@ -985,7 +991,8 @@
             _marker = db.gis_marker
             _projection = db.gis_projection
             have_tables = _config and _projection
-        except:
+        except Exception, exception:
+            self.debug(exception)
             have_tables = False
 
         row = None
@@ -1002,16 +1009,13 @@
             if not row:
                 if auth.is_logged_in():
                     # Read personalised config, if available.
-                    try:
-                        query = (db.pr_person.uuid == auth.user.person_uuid) & \
-                                (_config.pe_id == db.pr_person.pe_id) & \
-                                (_marker.id == _config.marker_id) & \
-                                (_projection.id == _config.projection_id)
-                        row = db(query).select(limitby=(0, 1)).first()
-                        if row:
-                            config_id = row["gis_config"].id
-                    except:
-                        pass
+                    query = (db.pr_person.uuid == auth.user.person_uuid) & \
+                            (_config.pe_id == db.pr_person.pe_id) & \
+                            (_marker.id == _config.marker_id) & \
+                            (_projection.id == _config.projection_id)
+                    row = db(query).select(limitby=(0, 1)).first()
+                    if row:
+                        config_id = row["gis_config"].id
                 if not row:
                     # No personal config or not logged in. Use site default.
                     config_id = 1
@@ -1148,7 +1152,7 @@
         if level:
             try:
                 return location_hierarchy[level]
-            except:
+            except KeyError:
                 return level
         else:
             return location_hierarchy
@@ -1197,7 +1201,8 @@
         if level:
             try:
                 return all_levels[level]
-            except:
+            except Exception, exception:
+                
                 return level
         else:
             return all_levels
@@ -1472,7 +1477,7 @@
                 represent = db(table.id == value).select(table.name,
                                                          cache=cache,
                                                          limitby=(0, 1)).first().name
-            except:
+            except: # @ToDo: provide specific exception
                 # Keep the default from earlier
                 pass
 
@@ -1512,24 +1517,21 @@
                     lat_max = location.lat_max
 
                 else:
-                    s3_debug("Location searched within isn't a Polygon!")
-                    session.error = T("Location searched within isn't a Polygon!")
+                    self.debug("Location searched within isn't a Polygon!")
                     return None
-        except:
+        except: # @ToDo: need specific exception
             wkt = location
             if (wkt.startswith("POLYGON") or wkt.startswith("MULTIPOLYGON")):
                 # ok
                 lon_min = None
             else:
-                s3_debug("This isn't a Polygon!")
-                session.error = T("This isn't a Polygon!")
+                self.debug("This isn't a Polygon!")
                 return None
 
         try:
             polygon = wkt_loads(wkt)
-        except:
-            s3_debug("Invalid Polygon!")
-            session.error = T("Invalid Polygon!")
+        except: # @ToDo: need specific exception
+            self.debug("Invalid Polygon!")
             return None
 
         table = db[tablename]
@@ -1537,8 +1539,7 @@
 
         if "location_id" not in table.fields():
             # @ToDo: Add any special cases to be able to find the linked location
-            s3_debug("This table doesn't have a location_id!")
-            session.error = T("This table doesn't have a location_id!")
+            self.debug("This table doesn't have a location_id!")
             return None
 
         query = (table.location_id == locations.id)
@@ -1570,7 +1571,10 @@
                         # Save Record
                         output.records.append(row)
                 except shapely.geos.ReadingError:
-                    s3_debug("Error reading wkt of location with id", row.id)
+                    self.debug(
+                        "Error reading wkt of location with id",
+                        value=row.id
+                    )
         else:
             # 1st check for Features included within the bbox (faster)
             def in_bbox(row):
@@ -1596,7 +1600,10 @@
                         # Save Record
                         output.records.append(row)
                 except shapely.geos.ReadingError:
-                    s3_debug("Error reading wkt of location with id", row.id)
+                    self.debug(
+                        "Error reading wkt of location with id",
+                        value = row.id,
+                    )
 
         return output
 
@@ -2072,38 +2079,41 @@
             current_row += 1
             try:
                 name0 = row.pop("ADM0_NAME")
-            except:
+            except KeyError:
                 name0 = ""
             try:
                 name1 = row.pop("ADM1_NAME")
-            except:
+            except KeyError:
                 name1 = ""
             try:
                 name2 = row.pop("ADM2_NAME")
-            except:
+            except KeyError:
                 name2 = ""
             try:
                 name3 = row.pop("ADM3_NAME")
-            except:
+            except KeyError:
                 name3 = ""
             try:
                 name4 = row.pop("ADM4_NAME")
-            except:
+            except KeyError:
                 name4 = ""
             try:
                 name5 = row.pop("ADM5_NAME")
-            except:
+            except KeyError:
                 name5 = ""
 
             if not name5 and not name4 and not name3 and \
                not name2 and not name1:
                 # We need a name! (L0's are already in DB)
-                s3_debug("No name provided", current_row)
+                s3_debug(
+                    "No name provided",
+                    current_row,
+                )
                 continue
 
             try:
                 wkt = row.pop("WKT")
-            except:
+            except KeyError:
                 wkt = None
                 try:
                     lat = row.pop("LAT")
@@ -2112,21 +2122,17 @@
                     lat = None
                     lon = None
 
-            if domain:
-                try:
-                    uuid = "%s/%s" % (domain,
-                                      row.pop("UUID"))
-                except:
-                    uuid = ""
+            try:
+                uuid = row.pop("UUID")
+            except KeyError:
+                uuid = ""
             else:
-                try:
-                    uuid = row.pop("UUID")
-                except:
-                    uuid = ""
+                if domain:
+                    uuid = "%s/%s" % (domain, uuid)
 
             try:
                 code = row.pop("CODE")
-            except:
+            except KeyError:
                 code = ""
 
             population = ""
@@ -2172,7 +2178,7 @@
             # Calculate Centroid & Bounds
             if wkt:
                 try:
-                    # Valid WKT
+                        # Valid WKT
                     shape = wkt_loads(wkt)
                     centroid_point = shape.centroid
                     lon = centroid_point.x
@@ -2186,7 +2192,7 @@
                         feature_type = 1 # Point
                     else:
                         feature_type = 3 # Polygon
-                except:
+                except: # @ToDo: provide specific exception
                     s3_debug("Invalid WKT", name)
                     continue
             else:
@@ -2326,7 +2332,7 @@
         else:
             cached = False
             if not os.access(cachepath, os.W_OK):
-                s3_debug("Folder not writable", cachepath)
+                self.debug("Folder not writable", cachepath)
                 return
 
         if not cached:
@@ -2335,11 +2341,11 @@
                 f = fetch(url)
             except (urllib2.URLError,):
                 e = sys.exc_info()[1]
-                s3_debug("URL Error", e)
+                self.debug("URL Error", e)
                 return
             except (urllib2.HTTPError,):
                 e = sys.exc_info()[1]
-                s3_debug("HTTP Error", e)
+                self.debug("HTTP Error", e)
                 return
 
             # Unzip File
@@ -2352,8 +2358,8 @@
                     # For now, 2.5 users need to download/unzip manually to cache folder
                     myfile.extract(filename, cachepath)
                     myfile.close()
-                except:
-                    s3_debug("Zipfile contents don't seem correct!")
+                except IOError:
+                    self.debug("Zipfile contents don't seem correct!")
                     myfile.close()
                     return
 
@@ -2469,7 +2475,7 @@
                             # Should be just a single parent
                             break
                     except shapely.geos.ReadingError:
-                        s3_debug("Error reading wkt of location with id", row.id)
+                        self.debug("Error reading wkt of location with id", row.id)
 
                 # Add entry to database
                 table.insert(uuid=uuid,
@@ -2489,7 +2495,7 @@
             else:
                 continue
 
-        s3_debug("All done!")
+        self.debug("All done!")
         return
 
     # -------------------------------------------------------------------------
@@ -2732,7 +2738,7 @@
 
         db = current.db
         in_bbox = self.query_features_by_bbox(*shape.bounds)
-        has_wkt = (db.gis_location.wkt != None) & (db.gis_location.wkt != '')
+        has_wkt = (db.gis_location.wkt != None) & (db.gis_location.wkt != "")
 
         for loc in db(in_bbox & has_wkt).select():
             try:
@@ -2740,7 +2746,7 @@
                 if location_shape.intersects(shape):
                     yield loc
             except shapely.geos.ReadingError:
-                s3_debug("Error reading wkt of location with id", loc.id)
+                self.debug("Error reading wkt of location with id", loc.id)
 
     # -------------------------------------------------------------------------
     def _get_features_by_latlon(self, lat, lon):
@@ -2796,7 +2802,7 @@
                 try :
                     shape = wkt_loads(location.wkt)
                 except:
-                    s3_debug("Error reading WKT", location.wkt)
+                    self.debug("Error reading WKT", location.wkt)
                     continue
                 bounds = shape.bounds
                 table[location.id] = dict(
@@ -2947,7 +2953,12 @@
             map_width = width
         else:
             map_width = config.map_width
-        if bbox and (-90 < bbox["max_lat"] < 90) and (-90 < bbox["min_lat"] < 90) and (-180 < bbox["max_lon"] < 180) and (-180 < bbox["min_lon"] < 180):
+        if (bbox 
+            and (-90 < bbox["max_lat"] < 90) 
+            and (-90 < bbox["min_lat"] < 90)
+            and (-180 < bbox["max_lon"] < 180)
+            and (-180 < bbox["min_lon"] < 180)
+        ):
             # We have sane Bounds provided, so we should use them
             pass
         else:
@@ -2971,15 +2982,21 @@
             projection = config.epsg
 
 
-        if projection != 900913 and projection != 4326:
+        if projection not in (900913, 4326):
             # Test for Valid Projection file in Proj4JS library
-            projpath = os.path.join(request.folder, "static", "scripts", "gis", "proj4js", "lib", "defs", "EPSG%s.js" % projection)
+            projpath = os.path.join(
+                request.folder, "static", "scripts", "gis", "proj4js", \
+                "lib", "defs", "EPSG%s.js" % projection
+            )
             try:
                 f = open(projpath, "r")
                 f.close()
             except:
-                session.error = "%s /static/scripts/gis/proj4js/lib/defs" % T("Projection not supported - please add definition to")
-                redirect(URL(c="gis", f="projection"))
+                session.error = "'%s' %s /static/scripts/gis/proj4js/lib/defs" % (
+                    projection,
+                    T("Projection not supported - please add definition to")
+                )
+                redirect(URL(r=request, c="gis", f="projection"))
 
         units = config.units
         maxResolution = config.maxResolution
@@ -3052,11 +3069,12 @@
         #########
         # Scripts
         #########
+        
         def add_javascript(script):
             if type(script) == SCRIPT:
                 html.append(script)
             elif script.startswith("http"):
-                 html.append(
+                html.append(
                     SCRIPT(_type="text/javascript",
                            _src=script))
             else:
@@ -3066,7 +3084,7 @@
 
         debug = session.s3.debug
         if debug:
-            if projection != 900913 and projection != 4326:
+            if projection not in (900913, 4326):
                 add_javascript("scripts/gis/proj4js/lib/proj4js-combined.js")
                 add_javascript("scripts/gis/proj4js/lib/defs/EPSG%s.js" % projection)
 
@@ -3079,7 +3097,7 @@
                 add_javascript("scripts/gis/usng2.js")
                 add_javascript("scripts/gis/MP.js")
         else:
-            if projection != 900913 and projection != 4326:
+            if projection not in (900913, 4326):
                 add_javascript("scripts/gis/proj4js/lib/proj4js-compressed.js")
                 add_javascript("scripts/gis/proj4js/lib/defs/EPSG%s.js" % projection)
             add_javascript("scripts/gis/OpenLayers.js")
@@ -3181,20 +3199,20 @@
         # If we do come back to it, then it should be moved to static
         if print_tool:
             url = print_tool["url"]
-            url+'' # check url can be concatenated with strings
+            url+"" # check url can be concatenated with strings
             if "title" in print_tool:
-                mapTitle = str(print_tool["mapTitle"])
+                mapTitle = unicode(print_tool["mapTitle"])
             else:
-                mapTitle = str(T("Map from Sahana Eden"))
+                mapTitle = unicode(T("Map from Sahana Eden"))
             if "subtitle" in print_tool:
-                subTitle = str(print_tool["subTitle"])
+                subTitle = unicode(print_tool["subTitle"])
             else:
-                subTitle = str(T("Printed from Sahana Eden"))
+                subTitle = unicode(T("Printed from Sahana Eden"))
             if session.auth:
-                creator = session.auth.user.email
+                creator = unicode(session.auth.user.email)
             else:
                 creator = ""
-            print_tool1 = "".join(("""
+            print_tool1 = u"".join(("""
         if (typeof(printCapabilities) != 'undefined') {
             // info.json from script headers OK
             printProvider = new GeoExt.data.PrintProvider({
@@ -3218,7 +3236,7 @@
             //    printProvider: printProvider
             //});
             // A layer to display the print page extent
-            //var pageLayer = new OpenLayers.Layer.Vector('""", str(T("Print Extent")), """');
+            //var pageLayer = new OpenLayers.Layer.Vector('""", unicode(T("Print Extent")), """');
             //pageLayer.addFeatures(printPage.feature);
             //pageLayer.setVisibility(false);
             //map.addLayer(pageLayer);
@@ -3234,7 +3252,7 @@
             //});
             // The form with fields controlling the print output
             S3.gis.printFormPanel = new Ext.form.FormPanel({
-                title: '""", str(T("Print Map")), """',
+                title: '""", unicode(T("Print Map")), """',
                 rootVisible: false,
                 split: true,
                 autoScroll: true,
@@ -3247,7 +3265,7 @@
                 defaults: {anchor: '100%%'},
                 listeners: {
                     'expand': function() {
-                        //if (null == mapPanel.map.getLayersByName('""", str(T("Print Extent")), """')[0]) {
+                        //if (null == mapPanel.map.getLayersByName('""", unicode(T("Print Extent")), """')[0]) {
                         //    mapPanel.map.addLayer(pageLayer);
                         //}
                         if (null == mapPanel.plugins[0]) {
@@ -3275,7 +3293,7 @@
                     xtype: 'textarea',
                     name: 'comment',
                     value: '',
-                    fieldLabel: '""", str(T("Comment")), """',
+                    fieldLabel: '""", unicode(T("Comment")), """',
                     plugins: new GeoExt.plugins.PrintPageField({
                         printPage: printPage
                     })
@@ -3283,7 +3301,7 @@
                     xtype: 'combo',
                     store: printProvider.layouts,
                     displayField: 'name',
-                    fieldLabel: '""", str(T("Layout")), """',
+                    fieldLabel: '""", T("Layout").decode("utf-8"), """',
                     typeAhead: true,
                     mode: 'local',
                     triggerAction: 'all',
@@ -3294,7 +3312,7 @@
                     xtype: 'combo',
                     store: printProvider.dpis,
                     displayField: 'name',
-                    fieldLabel: '""", str(T("Resolution")), """',
+                    fieldLabel: '""", unicode(T("Resolution")), """',
                     tpl: '<tpl for="."><div class="x-combo-list-item">{name} dpi</div></tpl>',
                     typeAhead: true,
                     mode: 'local',
@@ -3311,7 +3329,7 @@
                 //    xtype: 'combo',
                 //    store: printProvider.scales,
                 //    displayField: 'name',
-                //    fieldLabel: '""", str(T("Scale")), """',
+                //    fieldLabel: '""", unicode(T("Scale")), """',
                 //    typeAhead: true,
                 //    mode: 'local',
                 //    triggerAction: 'all',
@@ -3321,13 +3339,13 @@
                 //}, {
                 //    xtype: 'textfield',
                 //    name: 'rotation',
-                //    fieldLabel: '""", str(T("Rotation")), """',
+                //    fieldLabel: '""", unicode(T("Rotation")), """',
                 //    plugins: new GeoExt.plugins.PrintPageField({
                 //        printPage: printPage
                 //    })
                 }],
                 buttons: [{
-                    text: '""", str(T("Create PDF")), """',
+                    text: '""", unicode(T("Create PDF")), """',
                     handler: function() {
                         // the PrintExtent plugin is the mapPanel's 1st plugin
                         //mapPanel.plugins[0].print();
@@ -3345,7 +3363,7 @@
         } else {
             // Display error diagnostic
             S3.gis.printFormPanel = new Ext.Panel ({
-                title: '""", str(T("Print Map")), """',
+                title: '""", unicode(T("Print Map")), """',
                 rootVisible: false,
                 split: true,
                 autoScroll: true,
@@ -3356,7 +3374,7 @@
                 bodyStyle: 'padding:5px',
                 labelAlign: 'top',
                 defaults: {anchor: '100%'},
-                html: '""", str(T("Printing disabled since server not accessible")), """: <BR />""", url, """'
+                html: '""", unicode(T("Printing disabled since server not accessible")), """: <BR />""", unicode(url), """'
             });
         }
         """))
@@ -3442,40 +3460,40 @@
             name_safe = re.sub("'", "", layer.name)
             if layer.url2:
                 url2 = """,
-    url2: '%s'""" % layer.url2
+    "url2": "%s\"""" % layer.url2
             else:
                 url2 = ""
             if layer.url3:
                 url3 = """,
-    url3: '%s'""" % layer.url3
+    "url3": "%s\"""" % layer.url3
             else:
                 url3 = ""
             if layer.base:
                 base = ""
             else:
                 base = """,
-    isBaseLayer: false"""
+    "isBaseLayer": false"""
             if layer.visible:
                 visibility = ""
             else:
                 visibility = """,
-    visibility: false"""
+    "visibility": false"""
             if layer.attribution:
                 attribution = """,
-    attribution: '%s'""" % layer.attribution
+    "attribution": %s""" % repr(layer.attribution)
             else:
                 attribution = ""
             if layer.zoom_levels is not None and layer.zoom_levels != 19:
                 zoomLevels = """,
-    zoomLevels: %i""" % layer.zoom_levels
+    "zoomLevels": %i""" % layer.zoom_levels
             else:
                 zoomLevels = ""
 
             # Generate JS snippet to pass to static
             layers_osm += """
 S3.gis.layers_osm[%i] = {
-    name: '%s',
-    url1: '%s'%s%s%s%s%s%s
+    "name": "%s",
+    "url1": "%s"%s%s%s%s%s%s
 }
 """ % (counter,
        name_safe,
@@ -3487,6 +3505,7 @@
        attribution,
        zoomLevels)
 
+
         # ---------------------------------------------------------------------
         # XYZ
         # @ToDo: Migrate to Class/Static
@@ -3678,7 +3697,7 @@
 
             if "active" in layer and not layer["active"]:
                 visibility = """,
-    visibility: false"""
+    "visibility": false"""
             else:
                 visibility = ""
 
@@ -3709,32 +3728,33 @@
                         marker_url = ""
                 if marker_url:
                     markerLayer = """,
-    marker_url: '%s',
-    marker_height: %i,
-    marker_width: %i""" % (marker_url, marker_height, marker_width)
+    "marker_url": "%s",
+    "marker_height": %i,
+    "marker_width": %i""" % (marker_url, marker_height, marker_width)
 
             if "opacity" in layer and layer["opacity"] != 1:
                 opacity = """,
-    opacity: %.1f""" % layer["opacity"]
+    "opacity": %.1f""" % layer["opacity"]
             else:
                 opacity = ""
             if "cluster_distance" in layer and layer["cluster_distance"] != self.cluster_distance:
                 cluster_distance = """,
-    cluster_distance: %i""" % layer["cluster_distance"]
+    "cluster_distance": %i""" % layer["cluster_distance"]
             else:
                 cluster_distance = ""
             if "cluster_threshold" in layer and layer["cluster_threshold"] != self.cluster_threshold:
                 cluster_threshold = """,
-    cluster_threshold: %i""" % layer["cluster_threshold"]
+    "cluster_threshold": %i""" % layer["cluster_threshold"]
             else:
                 cluster_threshold = ""
 
             # Generate JS snippet to pass to static
             layers_feature_queries += """
 S3.gis.layers_feature_queries[%i] = {
-    name: '%s',
-    url: '%s'%s%s%s%s%s
-}""" % (counter,
+    "name": "%s",
+    "url": "%s"%s%s%s%s%s
+}
+""" % (counter,
         name,
         url,
         visibility,
@@ -3742,36 +3762,44 @@
         opacity,
         cluster_distance,
         cluster_threshold)
-
+        
         # ---------------------------------------------------------------------
         # Add Layers from the Catalogue
         # ---------------------------------------------------------------------
         layers_config = ""
         if catalogue_layers:
             for LayerType in [
-                    #OSMLayer,
-                    Bing,
-                    Google,
-                    Yahoo,
-                    TMSLayer,
-                    WMSLayer,
-                    FeatureLayer,
-                    GeoJSONLayer,
-                    GeoRSSLayer,
-                    GPXLayer,
-                    KMLLayer,
-                    WFSLayer
-                ]:
-                # Instantiate the Class
-                layer = LayerType(self)
-                layer_type_js = layer.as_javascript()
-                if layer_type_js:
-                    # Add to the output JS
-                    layers_config = "".join((layers_config,
-                                             layer_type_js))
-                    if layer.scripts:
-                        for script in layer.scripts:
-                            add_javascript(script)
+                #OSMLayer,
+                BingLayer,
+                GoogleLayer,
+                YahooLayer,
+                TMSLayer,
+                WMSLayer,
+                FeatureLayer,
+                GeoJSONLayer,
+                GeoRSSLayer,
+                GPXLayer,
+                KMLLayer,
+                WFSLayer
+            ]:
+                try:
+                    # Instantiate the Class
+                    layer = LayerType(self)
+                    layer_type_js = layer.as_javascript()
+                    if layer_type_js:
+                        # Add to the output JS
+                        layers_config = "".join((layers_config,
+                                                 layer_type_js))
+                        if layer.scripts:
+                            for script in layer.scripts:
+                                add_javascript(script)
+                except Exception, exception:
+                    if debug:
+                        raise
+                    else:
+                        session.warning.append(
+                            LayerType.__name__ + " not shown due to error"
+                        )                
 
             # -----------------------------------------------------------------
             # Coordinate Grid - only one possible
@@ -3845,6 +3873,7 @@
             "S3.gis.marker_default_width = %i;\n" % marker_default.width,
             osm_auth,
             layers_osm,
+            layers_feature_queries,
             _features,
             layers_config,
             # i18n Labels
@@ -3877,6 +3906,7 @@
         ))))
 
         # Static Script
+
         if debug:
             add_javascript("scripts/S3/s3.gis.js")
             add_javascript("scripts/S3/s3.gis.layers.js")
@@ -3886,7 +3916,6 @@
 
         # Dynamic Script (stuff which should, as far as possible, be moved to static)
         html.append(SCRIPT(layers_js + \
-                           #layers_xyz + \
                            print_tool1))
 
         # Set up map plugins
@@ -3901,8 +3930,7 @@
 
         return html
 
-
-# =============================================================================
+# -----------------------------------------------------------------------------
 class Marker(object):
     """ Represents a Map Marker """
     def __init__(self, gis, id=None):
@@ -3935,6 +3963,13 @@
         #self.url = URL(c="static", f="img",
         #               args=["markers", marker.image])
 
+    def add_attributes_to_output(self, output):
+        output.update(
+            marker_image = self.image,
+            marker_height = self.height,
+            marker_width = self.width,
+        )
+        
 # -----------------------------------------------------------------------------
 class Projection(object):
     """ Represents a Map Projection """
@@ -3960,41 +3995,34 @@
         self.epsg = projection.epsg
 
 # -----------------------------------------------------------------------------
+
+def config_dict(mandatory, defaulted):
+    d = dict(mandatory)
+    for key, (value, defaults) in defaulted.iteritems():
+        if value not in defaults:
+            d[key] = value
+    return d
+
+
+# the layer code only needs to do:
+# any database lookups to get extra data
+# security checks.
+
+# then it generates appropriate JSON strings.
+
 class Layer(object):
     """
-        Base Class for Layers
-            Not meant to be instantiated direct
+        Abstract Base Class for Layers
     """
-    def __init__(self, gis, record=None):
+    def __init__(self, gis):
+        self.gis = gis
         db = current.db
-
-        self.gis = gis
-        # This usually arrives later
-        self.record = record
-        # Ensure all attributes available (even if Null)
-        self._refresh()
-        self.scripts = []
-
-    def add_record(self, record):
-        """
-            Update the record & refresh the attributes
-        """
-        if record:
-            self.record = record
-            self._refresh()
-        else:
-            return
-
-    def as_dict(self):
-        """
-            Output the Layer as a Python dictionary
-            - this is used to build a JSON of the overall dict of layers
-        """
-        record = self.record
-        if record:
-            return record
-        else:
-            return
+        try:
+            self.table = db[self.table_name]
+        except:
+            current.manager.load(self.table_name)
+            self.table = db[tablename]
+
 
     def as_json(self):
         """
@@ -4006,674 +4034,502 @@
         else:
             return
 
-    def as_javascript(self):
-        """
-            Output the Layer as Javascript
-            - suitable for inclusion in the HTML page
-        """
-        gis = self.gis
-        auth = gis.auth
-        db = current.db
-        table = self.table
-
-        layer_type_list = []
-        # Read the enabled Layers
-        records = db(table.enabled == True).select()
-        for record in records:
-            # Check user is allowed to access the layer
-            role_required = record.role_required
-            if (not role_required) or auth.s3_has_role(role_required):
-                # Pass the record to the Class
-                self.add_record(record)
-                # Read the output dict for this layer
-                layer_dict = self.as_dict()
-                if layer_dict:
-                    # Add this layer to the list of layers for this layer type
-                    layer_type_list.append(layer_dict)
-
-        if layer_type_list:
-            # Output the Layer Type as JSON
-            layer_type_json = json.dumps(layer_type_list,
-                                         sort_keys=True,
-                                         indent=4)
-            layer_type_js = "".join(("%s = " % self.js_array,
-                                     layer_type_json,
-                                     "\n"))
-            return layer_type_js
-
-    def _name_safe(self, name):
-        """
-            Make the name safe for use in JSON
-            i.e. any Unicode character allowed except for " & \
-        """
-        return re.sub('[\\"]', "", name)
-
-    def _refresh(self):
-        " Refresh the attributes of the Layer "
-        table = self.table
-        if "marker_id" in table:
-            self.set_marker()
-        if "projection_id" in table:
-            self.set_projection()
-
-    def set_marker(self):
-        " Set the Marker for the Layer "
-        gis = self.gis
-        record = self.record
-        if record:
-            marker = Marker(gis, record.marker_id)
-            self.marker = marker
-        else:
-            self.marker = None
-
-    def set_projection(self):
-        " Set the Projection for the Layer "
-        gis = self.gis
-        record = self.record
-        if record:
-            projection = Projection(gis, record.projection_id)
-            self.projection = projection
-        else:
-            self.projection = None
-
-# -----------------------------------------------------------------------------
-class OneLayer(Layer):
-    """
-        Base Class for Layers with just a single record
-            Not meant to be instantiated direct
-    """
-
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = ""
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-        if not record:
-            # There is only ever 1 layer
-            record = db(table.id > 0).select().first()
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.OneLayer"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_javascript(self):
-        """
-            Output the Layer as Javascript
-            - suitable for inclusion in the HTML page
-        """
-        auth = self.gis.auth
-        record = self.record
-        # Check Layer exists in the DB
-        if not record:
-            return None
-        # Check Layer is enabled
-        if not record.enabled:
-            return None
-        # Check user is allowed to access the Layer
-        role_required = record.role_required
-        if role_required and not auth.s3_has_role(role_required):
-            return None
-        # Read the output JSON for this layer
-        layer_type_json = self.as_json()
-        layer_type_js = "".join(("%s = " % self.js_array,
-                                 layer_type_json,
-                                 "\n"))
-        return layer_type_js
-
-    def _set_api_key(self):
-        " Set the API Key for the Layer "
-        record = self.record
-        if record:
-            self.apikey = record.apikey
-        else:
-            self.apikey = None
-
-    def _refresh(self):
-        " Refresh the attributes of the Layer "
-        table = self.table
-        if "apikey" in table:
-            self._set_api_key()
-
-# -----------------------------------------------------------------------------
-class Bing(OneLayer):
-    """ Bing Layers from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_bing"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-        if not record:
-            # There is only ever 1 layer
-            record = db(table.id > 0).select().first()
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.Bing"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
+
+
+# -----------------------------------------------------------------------------
+class SingleRecordLayer(Layer):
+    """
+        Abstract Base Class for Layers with just a single record
+    """
+    
+    def __init__(self, gis):
+        super(SingleRecordLayer, self).__init__(gis)
+        table = self.table
+        records = current.db(table.id > 0).select()
+        assert len(records) <= 1, (
+            "There should only ever be 0 or 1 %s" % self.__class__.__name__
+        )
+        self.record = None
+        record = records.first()
+        if record is not None:
+            if record.enabled:
+                role_required = record.role_required
+                if not role_required or self.gis.auth.s3_has_role(role_required):
+                    self.record = record
+            # Refresh the attributes of the Layer
+            if "apikey" in table:
+                if record:
+                    self.apikey = record.apikey
+                else:
+                    self.apikey = None
+        self.gis = gis
+        self.scripts = []
+
+    def as_javascript(self):
+        """
+            Output the Layer as Javascript
+            - suitable for inclusion in the HTML page
+        """
+        if self.record:
+            if "apikey" in self.table and not self.apikey:
+                raise Exception("Cannot display a %s if we have no valid API Key" % self.__class__.__name__)
+            json = self.as_json()
+            if json:
+                return "%s = %s\n" % (
+                    self.js_array,
+                    json
+                )
+            else:
+                return None
+        else:
+            return None
+
+# -----------------------------------------------------------------------------
+class BingLayer(SingleRecordLayer):
+    """ Bing Layer from Catalogue """
+    table_name = "gis_layer_bing"
+    js_array = "S3.gis.Bing"
+    
     def as_dict(self):
         gis = self.gis
         record = self.record
-        apikey = self.apikey
-
-        if not apikey:
-            # Cannot display Bing layers if we have no valid API Key
-            return None
-
-        config = gis.get_config()
-        if Projection(gis, id=config.projection_id).epsg != 900913:
-            # Cannot display Bing layers unless we're using the
-            # Spherical Mercator Projection
-            return None
-
-        # Mandatory attributes
-        output = {
-                "ApiKey": self.apikey
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.aerial_enabled:
-            output["Aerial"] = record.aerial or "Bing Satellite"
-        if record.road_enabled:
-            output["Road"] = record.road or "Bing Roads"
-        if record.hybrid_enabled:
-            output["Hybrid"] = record.hybrid or "Bing Hybrid"
-
-        return output
+        if record is not None:
+            config = self.gis.get_config()
+            if Projection(gis, id=config.projection_id).epsg != 900913:
+                raise Exception("Cannot display Bing layers unless we're using the Spherical Mercator Projection")
+            else:
+                # Mandatory attributes
+                output = {
+                    "ApiKey": self.apikey
+                    }
+
+                # Attributes which are defaulted client-side if not set
+                if record.aerial_enabled:
+                    output["Aerial"] = record.aerial or "Bing Satellite"
+                if record.road_enabled:
+                    output["Road"] = record.road or "Bing Roads"
+                if record.hybrid_enabled:
+                    output["Hybrid"] = record.hybrid or "Bing Hybrid"
+                return output
+        else:
+            return None
 
 # -----------------------------------------------------------------------------
-class Google(OneLayer):
+class GoogleLayer(SingleRecordLayer):
     """
         Google Layers/Tools from Catalogue
     """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_google"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-        debug = current.session.s3.debug
-        if not record:
-            # There is only ever 1 layer
-            record = db(table.id > 0).select().first()
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.Google"
-        self.record = record
-        self._refresh()
-
-        if record:
+    table_name = "gis_layer_google"
+    js_array = "S3.gis.Google"
+    
+    def __init__(self, gis):
+        super(GoogleLayer, self).__init__(gis)
+        record = self.record
+        if record is not None:
+            debug = current.session.s3.debug
+            add_script = self.scripts.append
             if record.mapmaker_enabled or record.mapmakerhybrid_enabled:
                 # Need to use v2 API
                 # http://code.google.com/p/gmaps-api-issues/issues/detail?id=2349
-                self.scripts = ["http://maps.google.com/maps?file=api&v=2&key=%s" % self.apikey]
+                add_script("http://maps.google.com/maps?file=api&v=2&key=%s" % self.apikey)
             else:
                 # v3 API
-                self.scripts = ["http://maps.google.com/maps/api/js?v=3.2&sensor=false"]
+                add_script("http://maps.google.com/maps/api/js?v=3.2&sensor=false")
                 if debug and record.streetview_enabled:
-                    self.scripts.append("scripts/gis/gxp/widgets/GoogleStreetViewPanel.js")
+                    add_script("scripts/gis/gxp/widgets/GoogleStreetViewPanel.js")
             if record.earth_enabled:
-                self.scripts.append("http://www.google.com/jsapi?key=%s" % self.apikey)
-                self.scripts.append(SCRIPT("google && google.load('earth', '1');", _type="text/javascript"))
+                add_script("http://www.google.com/jsapi?key=%s" % self.apikey)
+                add_script(SCRIPT("google && google.load('earth', '1');", _type="text/javascript"))
                 if debug:
-                    self.scripts.append("scripts/gis/gxp/widgets/GoogleEarthPanel.js")
-        else:
-            self.scripts = []
+                    add_script("scripts/gis/gxp/widgets/GoogleEarthPanel.js")
 
     def as_dict(self):
         gis = self.gis
         T = current.T
         record = self.record
-        apikey = self.apikey
-
-        if not apikey and (record.mapmaker_enabled or record.mapmakerhybrid_enabled):
-            # Cannot display Google layers if we have no valid API Key
+        if record is not None:
+            config = gis.get_config()
+            if Projection(gis, id=config.projection_id).epsg != 900913:
+                if record.earth_enabled:
+                    # But the Google Earth panel can still be enabled
+                    return {
+                        "Earth": str(T("Switch to 3D"))
+                        }
+                else:
+                    raise Exception("Cannot display Google layers unless we're using the Spherical Mercator Projection")
+
+
+            # Mandatory attributes
+            #"ApiKey": self.apikey
+            output = {
+                }
+
+            # Attributes which are defaulted client-side if not set
+            if record.satellite_enabled:
+                output["Satellite"] = record.satellite or "Google Satellite"
+            if record.maps_enabled:
+                output["Maps"] = record.maps or "Google Maps"
+            if record.hybrid_enabled:
+                output["Hybrid"] = record.hybrid or "Google Hybrid"
+            if record.mapmaker_enabled:
+                output["MapMaker"] = record.mapmaker or "Google MapMaker"
+            if record.mapmakerhybrid_enabled:
+                output["MapMakerHybrid"] = record.mapmakerhybrid or "Google MapMaker Hybrid"
+            if record.earth_enabled:
+                output["Earth"] = str(T("Switch to 3D"))
+            if record.streetview_enabled and not (record.mapmaker_enabled or record.mapmakerhybrid_enabled):
+                # Streetview doesn't work with v2 API
+                output["StreetviewButton"] = str(T("Click where you want to open Streetview"))
+                output["StreetviewTitle"] = str(T("Street View"))
+
+            return output
+        else:
             return None
 
-        config = gis.get_config()
-        if Projection(gis, id=config.projection_id).epsg != 900913:
-            # Cannot display Google layers unless we're using the
-            # Spherical Mercator Projection
-            if record.earth_enabled:
-                # But the Google Earth panel can still be enabled
-                output = {
-                    "Earth": str(T("Switch to 3D"))
-                    }
-                return output
-            else:
-                return None
-
-        # Mandatory attributes
-        #"ApiKey": self.apikey
-        output = {
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.satellite_enabled:
-            output["Satellite"] = record.satellite or "Google Satellite"
-        if record.maps_enabled:
-            output["Maps"] = record.maps or "Google Maps"
-        if record.hybrid_enabled:
-            output["Hybrid"] = record.hybrid or "Google Hybrid"
-        if record.mapmaker_enabled:
-            output["MapMaker"] = record.mapmaker or "Google MapMaker"
-        if record.mapmakerhybrid_enabled:
-            output["MapMakerHybrid"] = record.mapmakerhybrid or "Google MapMaker Hybrid"
-        if record.earth_enabled:
-            output["Earth"] = str(T("Switch to 3D"))
-        if record.streetview_enabled and not (record.mapmaker_enabled or record.mapmakerhybrid_enabled):
-            # Streetview doesn't work with v2 API
-            output["StreetviewButton"] = str(T("Click where you want to open Streetview"))
-            output["StreetviewTitle"] = str(T("Street View"))
-
-        return output
-
 # -----------------------------------------------------------------------------
-class Yahoo(OneLayer):
+class YahooLayer(SingleRecordLayer):
     """
         Yahoo Layers from Catalogue
 
         NB This will stop working on 13 September 2011
         http://developer.yahoo.com/blogs/ydn/posts/2011/06/yahoo-maps-apis-service-closure-announcement-new-maps-offerings-coming-soon/
     """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_yahoo"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-        if not record:
-            # There is only ever 1 layer
-            record = db(table.id > 0).select().first()
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.Yahoo"
-        self.record = record
-        self._refresh()
-        if record:
-            self.scripts = ["http://api.maps.yahoo.com/ajaxymap?v=3.8&appid=%s" % self.apikey]
-        else:
-            self.scripts = []
+    js_array = "S3.gis.Yahoo"
+    table_name = "gis_layer_yahoo"
+    
+    def __init__(self, gis):
+        super(YahooLayer, self).__init__(gis)
+        if self.record:
+            self.scripts.append("http://api.maps.yahoo.com/ajaxymap?v=3.8&appid=%s" % self.apikey)
+        config = gis.get_config()
+        if Projection(gis, id=config.projection_id).epsg != 900913:
+            raise Exception("Cannot display Yahoo layers unless we're using the Spherical Mercator Projection")
 
     def as_dict(self):
-        gis = self.gis
         record = self.record
-        apikey = self.apikey
-
-        if not apikey:
-            # Cannot display Yahoo layers if we have no valid API Key
-            return None
-
-        config = gis.get_config()
-        if Projection(gis, id=config.projection_id).epsg != 900913:
-            # Cannot display Yahoo layers unless we're using the
-            # Spherical Mercator Projection
-            return None
-
-        # Mandatory attributes
-        #"ApiKey": self.apikey
-        output = {
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.satellite_enabled:
-            output["Satellite"] = record.satellite or "Yahoo Satellite"
-        if record.maps_enabled:
-            output["Maps"] = record.maps or "Yahoo Maps"
-        if record.hybrid_enabled:
-            output["Hybrid"] = record.hybrid or "Yahoo Hybrid"
-
-        return output
+        if record is not None:
+            # Mandatory attributes
+            #"ApiKey": self.apikey
+            output = {
+                }
+
+            # Attributes which are defaulted client-side if not set
+            if record.satellite_enabled:
+                output["Satellite"] = record.satellite or "Yahoo Satellite"
+            if record.maps_enabled:
+                output["Maps"] = record.maps or "Yahoo Maps"
+            if record.hybrid_enabled:
+                output["Hybrid"] = record.hybrid or "Yahoo Hybrid"
+
+            return output
+        else:
+            return None
+
+class MultiRecordLayer(Layer):
+    def __init__(self, gis):
+        super(MultiRecordLayer, self).__init__(gis)
+        self.sublayers = []
+        self.scripts = []
+
+        auth = gis.auth
+        
+        layer_type_list = []
+        # Read the enabled Layers
+        for record in current.db(self.table.enabled == True).select():
+            # Check user is allowed to access the layer
+            role_required = record.role_required
+            if (not role_required) or auth.s3_has_role(role_required):
+                self.sublayers.append(self.SubLayer(gis, record))
+
+    def as_javascript(self):
+        """
+            Output the Layer as Javascript
+            - suitable for inclusion in the HTML page
+        """
+        sublayer_dicts = []
+        for sublayer in self.sublayers:
+            # Read the output dict for this sublayer
+            sublayer_dict = sublayer.as_dict()
+            if sublayer_dict:
+                # Add this layer to the list of layers for this layer type
+                sublayer_dicts.append(sublayer_dict)
+        
+        if sublayer_dicts:
+            # Output the Layer Type as JSON
+            layer_type_json = json.dumps(sublayer_dicts,
+                                         sort_keys=True,
+                                         indent=4)
+            return "%s = %s\n" % (self.js_array, layer_type_json)
+        else:
+            return None
+
+    class SubLayer(object):
+        def __init__(self, gis, record):
+            # Ensure all attributes available (even if Null)
+            self.gis = gis
+            self.__dict__.update(record)
+            del record
+            self.safe_name = re.sub('[\\"]', "", self.name)
+            
+            if hasattr(self, "marker_id"):
+                self.marker = Marker(gis, self.marker_id)
+            if hasattr(self, "projection_id"):
+                self.projection = Projection(gis, self.projection_id)
+
+        def setup_clustering(self, output):
+            gis = self.gis
+            cluster_distance = gis.cluster_distance
+            cluster_threshold = gis.cluster_threshold
+            if self.cluster_distance != cluster_distance:
+                output["cluster_distance"] = self.cluster_distance
+            if self.cluster_threshold != cluster_threshold:
+                output["cluster_threshold"] = self.cluster_threshold
+                
+        def setup_visibility_and_opacity(self, output):
+            if not self.visible:
+                output["visibility"] = False
+            if self.opacity != 1:
+                output["opacity"] = "%.1f" % self.opacity
+
+        def add_attributes_if_not_default(self, output, **values_and_defaults):
+            # could also write values in debug mode, to check if defaults ignored.
+            # could also check values are not being overwritten.
+            for key, (value, defaults) in values_and_defaults.iteritems():
+                if value not in defaults:
+                    output[key] = value
+
+        #def set_marker(self):
+        #    " Set the Marker for the Layer "
+        #    gis = self.gis
+        #    self.marker = Marker(gis, self.marker_id)
+
+        #def set_projection(self):
+        #    " Set the Projection for the Layer "
+        #    gis = self.gis
+        #    self.projection = Projection(gis, self.projection_id)
 
 # -----------------------------------------------------------------------------
-class FeatureLayer(Layer):
+class FeatureLayer(MultiRecordLayer):
     """ Feature Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_feature"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.layers_features"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        gis = self.gis
-        cluster_distance = gis.cluster_distance
-        cluster_threshold = gis.cluster_threshold
-        record = self.record
-        marker = self.marker
-
-        auth = gis.auth
-        request = current.request
-        deployment_settings = gis.deployment_settings
-
-        if record.module not in deployment_settings.modules:
-            # Module is disabled
-            return
-        if not auth.permission(c=record.module, f=record.resource):
-            # User has no permission to this resource (in ACL)
-            return
-
-        name_safe = self._name_safe(record.name)
-
-        url = "%s.geojson?layer=%i" % (URL(record.module,record.resource),
-                                       record.id)
-        if record.filter:
-            url = "%s&%s" % (url, record.filter)
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
+    table_name = "gis_layer_feature"
+    js_array = "S3.gis.layers_features"
+
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def __init__(self, gis, record):
+            record_module = record.module
+            if record_module is not None:
+                if record_module not in gis.deployment_settings.modules:
+                    raise Exception("%s module is disabled" % record_module)
+                if not gis.auth.permission(c=record.module, f=record.resource):
+                    raise Exception("User has no permission to this resource (in ACL)")
+            else:
+                raise Exception("FeatureLayer Record '%s' has no module" % record.name)
+            super(FeatureLayer.SubLayer, self).__init__(gis, record)
+
+        def as_dict(self):
+            gis = self.gis
+
+            request = current.request
+            deployment_settings = gis.deployment_settings
+
+            url = "%s.geojson?layer=%i" % (URL(self.module, self.resource),
+                                           self.id)
+            if self.filter:
+                url = "%s&%s" % (url, self.filter)
+
+            # Mandatory attributes
+            output = {
+                "name": self.safe_name,
                 "url": url,
-                "marker_image": marker.image,
-                "marker_height": marker.height,
-                "marker_width": marker.width,
             }
-
-        # Attributes which are defaulted client-side if not set
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if record.cluster_distance != cluster_distance:
-            output["cluster_distance"] = record.cluster_distance
-        if record.cluster_threshold != cluster_threshold:
-            output["cluster_threshold"] = record.cluster_threshold
-
-        return output
+            self.marker.add_attributes_to_output(output)
+            self.setup_visibility_and_opacity(output)
+            self.setup_clustering(output)
+
+            return output
 
 # -----------------------------------------------------------------------------
-class GeoJSONLayer(Layer):
+class GeoJSONLayer(MultiRecordLayer):
     """ GeoJSON Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_geojson"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.layers_geojson"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        gis = self.gis
-        cluster_distance = gis.cluster_distance
-        cluster_threshold = gis.cluster_threshold
-        record = self.record
-        marker = self.marker
-        projection = self.projection
-
-        name_safe = self._name_safe(record.name)
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": record.url,
-                "marker_image": marker.image,
-                "marker_height": marker.height,
-                "marker_width": marker.width,
+    table_name = "gis_layer_geojson"
+    js_array = "S3.gis.layers_geojson"
+    
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            # Mandatory attributes
+            output = {
+                "name": self.safe_name,
+                "url": self.url,
             }
-
-        # Attributes which are defaulted client-side if not set
-        if projection.epsg != 4326:
-            output["projection"] = projection.epsg
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if record.cluster_distance != cluster_distance:
-            output["cluster_distance"] = record.cluster_distance
-        if record.cluster_threshold != cluster_threshold:
-            output["cluster_threshold"] = record.cluster_threshold
-
-        return output
+            self.marker.add_attributes_to_output(output)
+
+            # Attributes which are defaulted client-side if not set
+            projection = self.projection
+            if projection.epsg != 4326:
+                output["projection"] = projection.epsg
+            self.setup_visibility_and_opacity(output)
+            self.setup_clustering(output)
+
+            return output
 
 # -----------------------------------------------------------------------------
-class GeoRSSLayer(Layer):
+class GeoRSSLayer(MultiRecordLayer):
     """ GeoRSS Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_georss"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.cachetable = db.gis_cache
-        self.js_array = "S3.gis.layers_georss"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        gis = self.gis
-        cluster_distance = gis.cluster_distance
-        cluster_threshold = gis.cluster_threshold
-        record = self.record
-        marker = self.marker
-
-        db = current.db
-        request = current.request
-        public_url = gis.public_url
-        cachetable = self.cachetable
-
-        url = record.url
-        # Check to see if we should Download layer to the cache
-        download = True
-        query = (cachetable.source == url)
-        cached = db(query).select(cachetable.modified_on,
-                                  limitby=(0, 1)).first()
-        refresh = record.refresh or 900 # 15 minutes set if we have no data (legacy DB)
-        if cached:
-            modified_on = cached.modified_on
-            cutoff = modified_on + timedelta(seconds=refresh)
-            if request.utcnow < cutoff:
-                download = False
-        if download:
-            # Download layer to the Cache
-            # @ToDo: Call directly without going via HTTP
-            #        s3mgr = current.manager
-            # @ToDo: Make this async by using Celery (also use this for the refresh time)
-            fields = ""
-            if record.data:
-                fields = "&data_field=%s" % record.data
-            if record.image:
-                fields = "%s&image_field=%s" % (fields, record.image)
-            _url = "%s%s/update.georss?fetchurl=%s%s" % (public_url,
-                                                         URL(c="gis", f="cache_feed"),
-                                                         url,
-                                                         fields)
-            try:
-                # @ToDo: Need to commit to not have DB locked with SQLite?
-                fetch(_url)
-                if cached:
-                    # Clear old records which are no longer active
-                    query = (cachetable.source == url) & \
-                            (cachetable.modified_on < cutoff)
-                    db(query).delete()
-            except:
-                # Feed down
-                if cached:
-                    # Use cached copy
-                    # Should we Update timestamp to prevent every
-                    # subsequent request attempting the download?
-                    #query = (cachetable.source == url)
-                    #db(query).update(modified_on=request.utcnow)
-                    pass
-                else:
-                    # No cached copy available - skip layer
-                    return
-
-        name_safe = self._name_safe(record.name)
-
-        # Pass the GeoJSON URL to the client
-        # Filter to the source of this feed
-        url = "%s.geojson?cache.source=%s" % (URL(c="gis", f="cache_feed"),
-                                              url)
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": url,
-                "marker_image": marker.image,
-                "marker_height": marker.height,
-                "marker_width": marker.width,
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.refresh != 900:
-            output["refresh"] = record.refresh
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if record.cluster_distance != cluster_distance:
-            output["cluster_distance"] = record.cluster_distance
-        if record.cluster_threshold != cluster_threshold:
-            output["cluster_threshold"] = record.cluster_threshold
-
-        return output
+    table_name = "gis_layer_georss"
+    js_array = "S3.gis.layers_georss"
+    
+    def __init__(self, gis):
+        super(GeoRSSLayer, self).__init__(gis)
+        GeoRSSLayer.SubLayer.cachetable = current.db.gis_cache
+
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            gis = self.gis
+            
+            db = current.db
+            request = current.request
+            public_url = gis.public_url
+            cachetable = self.cachetable
+            
+            url = self.url
+            # Check to see if we should Download layer to the cache
+            download = True
+            query = (cachetable.source == url)
+            existing_cached_copy = db(query).select(cachetable.modified_on,
+                                      limitby=(0, 1)).first()
+            refresh = self.refresh or 900 # 15 minutes set if we have no data (legacy DB)
+            if existing_cached_copy:
+                modified_on = existing_cached_copy.modified_on
+                cutoff = modified_on + timedelta(seconds=refresh)
+                if request.utcnow < cutoff:
+                    download = False
+            if download:
+                # Download layer to the Cache
+                # @ToDo: Call directly without going via HTTP
+                #        s3mgr = current.manager
+                # @ToDo: Make this async by using Celery (also use this for the refresh time)
+                fields = ""
+                if self.data:
+                    fields = "&data_field=%s" % self.data
+                if self.image:
+                    fields = "%s&image_field=%s" % (fields, self.image)
+                _url = "%s%s/update.georss?fetchurl=%s%s" % (public_url,
+                                                             URL(c="gis", f="cache_feed"),
+                                                             url,
+                                                             fields)
+                try:
+                    # @ToDo: Need to commit to not have DB locked with SQLite?
+                    fetch(_url)
+                    if existing_cached_copy:
+                        # Clear old selfs which are no longer active
+                        query = (cachetable.source == url) & \
+                                (cachetable.modified_on < cutoff)
+                        db(query).delete()
+                except:
+                    # Feed down
+                    if existing_cached_copy:
+                        # Use cached copy
+                        # Should we Update timestamp to prevent every
+                        # subsequent request attempting the download?
+                        #query = (cachetable.source == url)
+                        #db(query).update(modified_on=request.utcnow)
+                        pass
+                    else:
+                        raise Exception("No cached copy available - skip layer")
+
+            name_safe = self.safe_name
+
+            # Pass the GeoJSON URL to the client
+            # Filter to the source of this feed
+            url = "%s.geojson?cache.source=%s" % (URL(c="gis", f="cache_feed"),
+                                                  url)
+
+            # Mandatory attributes
+            output = {
+                    "name": name_safe,
+                    "url": url,
+                }
+            self.marker.add_attributes_to_output(output)
+
+            # Attributes which are defaulted client-side if not set
+            if self.refresh != 900:
+                output["refresh"] = self.refresh
+            self.setup_visibility_and_opacity(output)
+            self.setup_clustering(output)
+
+            return output
 
 # -----------------------------------------------------------------------------
-class GPXLayer(Layer):
+class GPXLayer(MultiRecordLayer):
     """ GPX Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_gpx"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.layers_gpx"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        gis = self.gis
-        cluster_distance = gis.cluster_distance
-        cluster_threshold = gis.cluster_threshold
-        record = self.record
-        marker = self.marker
-
-        name_safe = self._name_safe(record.name)
-
-        request = current.request
-        url = URL(c="default", f="download",
-                  args=record.track)
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": url,
-                "marker_image": marker.image,
-                "marker_height": marker.height,
-                "marker_width": marker.width,
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if not record.waypoints:
-            output["waypoints"] = False
-        if not record.tracks:
-            output["tracks"] = False
-        if not record.routes:
-            output["routes"] = False
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if record.cluster_distance != cluster_distance:
-            output["cluster_distance"] = record.cluster_distance
-        if record.cluster_threshold != cluster_threshold:
-            output["cluster_threshold"] = record.cluster_threshold
-
-        return output
-
-    def refresh(self):
-        " Refresh the attributes of the Layer "
-        gis = self.gis
-        request = current.request
-        record = self.record
-        table = self.table
-        if "marker_id" in table:
-            self.set_marker()
-        if "projection_id" in table:
-            self.set_projection()
-        if record:
-            self.url = "%s/%s" % (URL(c="default", f="download"),
-                                  record.track)
-        else:
-            self.url = None
+    table_name = "gis_layer_gpx"
+    js_array = "S3.gis.layers_gpx"
+    
+    def __init__(self, gis):
+        super(GPXLayer, self).__init__(gis)
+
+#        if record:
+#            self.url = "%s/%s" % (URL(c="default", f="download"),
+#                                  record.track)
+#        else:
+#            self.url = None
+
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            gis = self.gis
+            request = current.request
+
+            url = URL(c="default", f="download",
+                      args=self.track)
+            
+            # Mandatory attributes
+            output = {
+                    "name": self.safe_name,
+                    "url": url,
+                }
+            self.marker.add_attributes_to_output(output)
+            self.add_attributes_if_not_default(
+                output,
+                waypoints = (self.waypoints, (True,)),
+                tracks = (self.tracks, (True,)),
+                routes = (self.routes, (True,)),
+            )
+            self.setup_visibility_and_opacity(output)
+            self.setup_clustering(output)
+            return output
 
 # -----------------------------------------------------------------------------
-class KMLLayer(Layer):
+class KMLLayer(MultiRecordLayer):
     """ KML Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_kml"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        # @ToDo: Migrate to gis_cache
-        self.cachetable = db.gis_cache2
-        self.js_array = "S3.gis.layers_kml"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
+    table_name = "gis_layer_kml"
+    js_array = "S3.gis.layers_kml"
+    
+    def __init__(self, gis):
+        super(KMLLayer, self).__init__(gis)
+        
+        "Set up the KML cache, should be done once per request"
         # Can we cache downloaded KML feeds?
         # Needed for unzipping & filtering as well
         # @ToDo: Should we move this folder to static to speed up access to cached content?
         #           Do we need to secure it?
         cachepath = os.path.join(current.request.folder, "uploads", "gis_cache")
-        if os.access(cachepath, os.W_OK):
-            cacheable = True
+
+        if os.path.exists(cachepath):
+            cacheable = os.access(cachepath, os.W_OK)
         else:
             try:
                 os.mkdir(cachepath)
+            except OSError, os_error:
+                self.gis.debug(
+                    "GIS: KML layers cannot be cached: %s %s" % (
+                        cachepath,
+                        os_error
+                    )
+                )
+                cacheable = False
+            else:
                 cacheable = True
+<<<<<<< TREE
             except:
                 cacheable = False
         self.cacheable = cacheable
@@ -4720,209 +4576,164 @@
                 # Download file (async, if workers alive)
                 current.s3task.async("download_kml",
                                      args=[record.id, filename])
+=======
+        # @ToDo: Migrate to gis_cache
+        KMLLayer.cachetable = current.db.gis_cache2
+        KMLLayer.cacheable = cacheable
+        KMLLayer.cachepath = cachepath
+        
+        
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            gis = self.gis
+
+            T = current.T
+            db = current.db
+            request = current.request
+            response = current.response
+            public_url = gis.public_url
+            
+            cachetable = KMLLayer.cachetable
+            cacheable = KMLLayer.cacheable
+            cachepath = KMLLayer.cachepath
+
+            name = self.name
+            if cacheable:
+                _name = urllib2.quote(name)
+                _name = _name.replace("%", "_")
+                filename = "%s.file.%s.kml" % (cachetable._tablename,
+                                               _name)
+
+                # Should we download a fresh copy of the source file?
+                download = True
+                query = (cachetable.name == name)
+                cached = db(query).select(cachetable.modified_on,
+                                          limitby=(0, 1)).first()
+                refresh = self.refresh or 900 # 15 minutes set if we have no data (legacy DB)
+>>>>>>> MERGE-SOURCE
                 if cached:
-                    db(query).update(modified_on=request.utcnow)
-                else:
-                    cachetable.insert(name=name, file=filename)
-
-            url = URL(c="default", f="download",
-                      args=[filename])
-        else:
-            # No caching possible (e.g. GAE), display file direct from remote (using Proxy)
-            # (Requires OpenLayers.Layer.KML to be available)
-            url = record.url
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": url,
-                "marker_image": marker.image,
-                "marker_height": marker.height,
-                "marker_width": marker.width,
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.title and record.title != "name":
-            output["title"] = record.title
-        if record.body and record.body != "description":
-            output["body"] = record.body
-        if record.refresh != 900:
-            output["refresh"] = record.refresh
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if record.cluster_distance != cluster_distance:
-            output["cluster_distance"] = record.cluster_distance
-        if record.cluster_threshold != cluster_threshold:
-            output["cluster_threshold"] = record.cluster_threshold
-
-        return output
+                    modified_on = cached.modified_on
+                    cutoff = modified_on + timedelta(seconds=refresh)
+                    if request.utcnow < cutoff:
+                        download = False
+
+                if download:
+                    # Download file
+                    if response.s3.tasks_active():
+                        # Async call
+                        db.task_scheduled.insert(name="download_kml_%s" % uuid.uuid4(),
+                                                 func="download_kml",
+                                                 args=json.dumps([record.id,
+                                                                  filename]))
+                    else:
+                        # Sync call
+                        gis.download_kml(self.id, filename)
+                    if cached:
+                        db(query).update(modified_on=request.utcnow)
+                    else:
+                        cachetable.insert(name=name, file=filename)
+
+                url = URL(r=request, c="default", f="download",
+                          args=[filename])
+            else:
+                # No caching possible (e.g. GAE), display file direct from remote (using Proxy)
+                # (Requires OpenLayers.Layer.KML to be available)
+                url = self.url
+
+            output = dict(
+                name = self.safe_name,
+                url = url,
+            )
+            self.add_attributes_if_not_default(
+                output,
+                title = (self.title, ("name", None, "")),
+                body = (self.body, ("description", None)),
+                refresh = (self.refresh, (900,)),
+            )
+            self.setup_visibility_and_opacity(output)
+            self.setup_clustering(output)
+            self.marker.add_attributes_to_output(output)
+            return output
 
 # -----------------------------------------------------------------------------
-class TMSLayer(Layer):
+class TMSLayer(MultiRecordLayer):
     """ TMS Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_tms"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.layers_tms"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        #gis = self.gis
-        record = self.record
-
-        name_safe = self._name_safe(record.name)
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": record.url,
-                "layername": record.layername
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.url2:
-            output["url2"] = record.url2
-        if record.url3:
-            output["url3"] = record.url3
-        if record.img_format != "png":
-             output["format"] = record.img_format
-        if record.zoom_levels != 9:
-            output["zoomLevels"] = record.zoom_levels
-        if record.attribution:
-            output["attribution"] = record.attribution
-
-        return output
+    table_name = "gis_layer_tms"
+    js_array = "S3.gis.layers_tms"
+
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            output = {
+                    "name": self.safe_name,
+                    "url": self.url,
+                    "layername": self.layername
+                }
+            self.add_attributes_if_not_default(
+                output,
+                url2 = (self.url2, (None,)),
+                url3 = (self.url3, (None,)),
+                format = (self.img_format, ("png", None)),
+                zoomLevels = (self.zoom_levels, (9,)),
+                attribution = (self.attribution, (None,)),
+            )
+            return output
 
 # -----------------------------------------------------------------------------
-class WFSLayer(Layer):
+class WFSLayer(MultiRecordLayer):
     """ WFS Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_wfs"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.layers_wfs"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        gis = self.gis
-        cluster_distance = gis.cluster_distance
-        cluster_threshold = gis.cluster_threshold
-        record = self.record
-        projection = self.projection
-
-        name_safe = self._name_safe(record.name)
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": record.url,
-                "title": record.title,
-                "featureType": record.featureType,
-                "featureNS": record.featureNS,
-                "schema": record.wfs_schema,
+    table_name = "gis_layer_wfs"
+    js_array = "S3.gis.layers_wfs"
+    
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            output = dict(
+                name = self.safe_name,
+                url = self.url,
+                title = self.title,
+                featureType = self.featureType,
+                featureNS = self.featureNS,
+                schema = self.wfs_schema,
+            )
+            self.add_attributes_if_not_default(
+                output,
+                version = (self.version, ("1.1.0",)),
+                geometryName = (self.geometryName, ("the_geom",)),
+                styleField = (self.style_field, (None,)),
+                styleValues = (self.style_values, ("{}", None)),
+                projection = (self.projection.epsg, (4326,)),
                 #editable
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if record.version != "1.1.0":
-            output["version"] = record.version
-        if record.geometryName != "the_geom":
-            output["geometryName"] = record.geometryName
-        if record.style_field:
-            output["styleField"] = record.style_field
-        if record.style_values and record.style_values != "{}":
-            output["styleValues"] = record.style_values
-        if projection.epsg != 4326:
-            output["projection"] = projection.epsg
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if record.cluster_distance != cluster_distance:
-            output["cluster_distance"] = record.cluster_distance
-        if record.cluster_threshold != cluster_threshold:
-            output["cluster_threshold"] = record.cluster_threshold
-
-        return output
+            )
+            self.setup_visibility_and_opacity(output)
+            self.setup_clustering(output)
+            return output
 
 # -----------------------------------------------------------------------------
-class WMSLayer(Layer):
+class WMSLayer(MultiRecordLayer):
     """ WMS Layer from Catalogue """
-    def __init__(self, gis, record=None):
-        db = current.db
-        tablename = "gis_layer_wms"
-        try:
-            table = db[tablename]
-        except:
-            current.manager.load(tablename)
-            table = db[tablename]
-
-        self.gis = gis
-        self.table = table
-        self.js_array = "S3.gis.layers_wms"
-        self.record = record
-        self._refresh()
-        self.scripts = []
-
-    def as_dict(self):
-        #gis = self.gis
-        record = self.record
-
-        name_safe = self._name_safe(record.name)
-
-        # Mandatory attributes
-        output = {
-                "name": name_safe,
-                "url": record.url,
-                "layers": record.layers
-            }
-
-        # Attributes which are defaulted client-side if not set
-        if not record.visible:
-            output["visibility"] = False
-        if record.opacity != 1:
-            output["opacity"] = "%.1f" % record.opacity
-        if not record.transparent:
-             output["transparent"] = False
-        if record.version != "1.1.1":
-             output["version"] = record.version
-        if record.img_format != "image/png":
-             output["format"] = record.img_format
-        if record.map:
-             output["map"] = record.map
-        if record.style:
-             output["style"] = record.style
-        if record.bgcolor:
-             output["bgcolor"] = record.bgcolor
-        if record.tiled:
-            output["tiled"] = True
-        if record.buffer:
-            output["buffer"] = record.buffer
-        if record.base:
-            output["base"] = True
-
-        return output
+    js_array = "S3.gis.layers_wms"
+    table_name = "gis_layer_wms"
+
+    class SubLayer(MultiRecordLayer.SubLayer):
+        def as_dict(self):
+            output = dict(
+                name = self.safe_name,
+                url = self.url,
+                layers = self.layers                
+            )
+            self.add_attributes_if_not_default(
+                output,
+                transparent = (self.transparent, (True,)),
+                version = (self.version, ("1.1.1",)),
+                format = (self.img_format, ("image/png",)),
+                map = (self.map, (None,)),
+                buffer = (self.buffer, (0,)),
+                base = (self.base, (False,)),
+                style = (self.style, (None,)),
+                bgcolor = (self.bgcolor, (None,)),
+                tiled = (self.tiled, (False, )),
+            )
+            self.setup_visibility_and_opacity(output)
+            return output
 
 # =============================================================================
 class S3MAP(S3Method):
@@ -5031,4 +4842,3 @@
         return page
 
 # END =========================================================================
-

=== added file 'modules/test_utils/AddedRole.py'
--- modules/test_utils/AddedRole.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/AddedRole.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,25 @@
+
+class AddedRole(object):
+    """Adds a role and removes it at the end of a test no matter what happens.
+    
+    """
+    def __init__(self, session, role):
+        self.role = role
+        self.session = session
+    
+    def __enter__(self):
+        roles = self.session.s3.roles
+        role = self.role
+        if not role in roles:
+            roles.append(role)
+
+    def __exit__(self, type, value, traceback):
+        session_s3_roles = self.session.s3.roles
+        roles = list(session_s3_roles)
+        for i in range(len(roles)):
+            session_s3_roles.pop(0)
+        add_role = session_s3_roles.append
+        role = self.role
+        for role in roles:
+            if role is not role:
+                add_role(role)

=== added file 'modules/test_utils/Change.py'
--- modules/test_utils/Change.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/Change.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,25 @@
+
+_UNDEFINED = object()
+
+class Change(object):
+    def __init__(self, target, changes):
+        self.changes = changes
+        self.target = target
+    
+    def __enter__(self):
+        assert not hasattr(self, "originals")
+        self.originals = originals = {}
+        # store originals and set new values
+        for name, value in self.changes.iteritems():
+            originals[name] = getattr(self.target, name, _UNDEFINED)
+            setattr(self.target, name, value)
+        
+    def __exit__(self, type, value, traceback):
+        # restore originals
+        for name, value in self.originals.iteritems():
+            if value is _UNDEFINED:
+                delattr(self, name)
+            else:
+                setattr(self.target, name, value)    
+        del self.originals
+    

=== added file 'modules/test_utils/ExpectSessionWarning.py'
--- modules/test_utils/ExpectSessionWarning.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/ExpectSessionWarning.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,14 @@
+
+class ExpectSessionWarning(object):
+    def __init__(self, session, warning):
+        self.warning = warning
+        self.session = session
+
+    def __enter__(self):
+        session = self.session
+        warnings = []
+        self.warnings = session.warning = warnings
+
+    def __exit__(self, type, value, traceback):
+        if type is None:
+            assert self.warning in self.warnings

=== added file 'modules/test_utils/ExpectedException.py'
--- modules/test_utils/ExpectedException.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/ExpectedException.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,13 @@
+
+class ExpectedException(object):
+    def __init__(self, ExceptionClass):
+        self.ExceptionClass = ExceptionClass
+    
+    def __enter__(self):
+        pass
+
+    def __exit__(self, type, value, traceback):
+        return issubclass(type, self.ExceptionClass), (
+            "%s not raised" % self.ExceptionClass.__name__
+        )
+

=== added file 'modules/test_utils/InsertedRecord.py'
--- modules/test_utils/InsertedRecord.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/InsertedRecord.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,19 @@
+
+from clear_table import clear_table
+
+class InsertedRecord(object):
+    """Inserts and commits a record and removes it at the end of
+    a test no matter what happens.
+    
+    """
+    def __init__(self, db, table, data):
+        self.db = db
+        self.table = table
+        self.data = data
+    
+    def __enter__(self):
+        self.table.insert(**self.data)
+        self.db.commit()
+        
+    def __exit__(self, type, value, traceback):
+        clear_table(self.db, self.table)

=== added file 'modules/test_utils/Web2pyNosePlugin.py'
--- modules/test_utils/Web2pyNosePlugin.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/Web2pyNosePlugin.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,106 @@
+
+import nose
+import re
+from itertools import imap
+import unittest
+
+class Web2pyNosePlugin(nose.plugins.base.Plugin):
+    # see: http://somethingaboutorange.com/mrl/projects/nose/0.11.1/plugins/writing.html
+    
+    """This plugin is designed to give the web2py environment to the tests.
+    """
+    score = 0
+    # always enable as this plugin can only 
+    # be selected by running this script
+    enabled = True
+    
+    def __init__(
+        self,
+        application_name,
+        environment,
+        directory_pattern,
+        test_folders
+    ):
+        super(Web2pyNosePlugin, self).__init__()
+        self.application_name = application_name
+        self.environment = environment
+        self.directory_pattern = directory_pattern
+        self.test_folders = test_folders
+        
+    def options(self, parser, env):
+        """Register command line options"""
+        pass
+        
+    def wantDirectory(self, dirname):
+        return bool(re.search(self.directory_pattern, dirname))
+
+    def wantFile(self, file_name):
+        print file_name
+        return file_name.endswith(".py") and any(
+            imap(file_name.__contains__, self.test_folders)
+        )
+
+    def wantModule(self, module):
+        return False
+
+    def loadTestsFromName(self, file_name, discovered):
+        """Sets up the unit-testing environment.
+        
+        This involves loading modules as if by web2py.
+        Also we must have a test database.
+        
+        If testing controllers, tests need to set up the request themselves.
+                
+        """
+        if file_name.endswith(".py"):
+            
+            # Is it possible that the module could load 
+            # other code that is using the original db?
+            
+            test_globals = self.environment
+
+            module_globals = dict(self.environment)
+            # execfile is used because it doesn't create a module
+            # or load the module from sys.modules if it exists.
+            
+            execfile(file_name, module_globals)
+            
+            import inspect
+            # we have to return something, otherwise nose
+            # will let others have a go, and they won't pass
+            # in the web2py environment, so we'll get errors
+            tests = []
+    
+            for name, thing in module_globals.iteritems():
+                if (
+                    # don't bother with globally imported things 
+                    name not in test_globals \
+                    # unless they have been overridden
+                    or test_globals[name] is not thing
+                ):
+                    if (
+                        isinstance(thing, type)
+                        and issubclass(thing, unittest.TestCase)
+                    ):
+                        # look for test methods
+                        for member_name in dir(thing):
+                            if member_name.startswith("test"):
+                                if callable(getattr(thing, member_name)):
+                                    tests.append(thing(member_name))
+                    elif (
+                        name.startswith("test") 
+                        or name.startswith("Test")
+                    ):
+                        if inspect.isfunction(thing):
+                            function = thing
+                            function_name = name
+                            # things coming from execfile have no module
+                            #print file_name, function_name, function.__module__
+                            if function.__module__ in ("__main__", None):
+                                tests.append(
+                                    nose.case.FunctionTestCase(function)
+                                )
+            return tests
+        else:
+            return []
+        

=== modified file 'modules/test_utils/__init__.py'
--- modules/test_utils/__init__.py	2011-08-11 19:25:52 +0000
+++ modules/test_utils/__init__.py	2011-09-06 11:51:25 +0000
@@ -1,2 +1,12 @@
 
-from compare_lines import compare_lines
\ No newline at end of file
+from compare_lines import compare_lines
+from clear_table import clear_table
+from find_JSON_format_data_structure import *
+from Web2pyNosePlugin import Web2pyNosePlugin
+from assert_equal import *
+
+from InsertedRecord import *
+from AddedRole import *
+from ExpectedException import *
+from Change import *
+from ExpectSessionWarning import ExpectSessionWarning

=== added file 'modules/test_utils/assert_equal.py'
--- modules/test_utils/assert_equal.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/assert_equal.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,60 @@
+
+
+def assert_same_type(expected, actual):
+    assert isinstance(actual, type(expected)), "%s vs. %s" % (type(expected), type(actual))
+
+def assert_equal_sequence(expected, actual):
+    assert len(expected) == len(actual), "length should be %i, not %i:\n%s" % (
+        len(expected), len(actual), actual
+    )
+    for i in range(len(expected)):
+        try:
+            assert_equal(expected[i], actual[i])
+        except AssertionError, assertion_error:
+            raise AssertionError(
+                str(assertion_error)
+            )
+
+def assert_equal_set(expected, actual):
+    missing = expected.difference(actual)
+    assert not missing, "Missing: %s" % ", ".join(missing)
+
+    extra = actual.difference(expected)
+    assert not extra, "Extra: %s" % ", ".join(extra)    
+
+def assert_equal_dict(expected, actual):
+    assert_equal_set(
+        expected = set(expected.keys()),
+        actual = set(actual.keys())
+    )    
+    for key in expected.iterkeys():
+        try:
+            assert_equal(expected[key], actual[key])
+        except AssertionError, assertion_error:
+            raise AssertionError(
+                "[%s] %s" % (
+                    key,
+                    str(assertion_error),
+                )
+            )
+
+def assert_equal_value(expected, actual):
+    assert expected == actual, "%s != %s" % (expected, actual)
+
+_compare_procs = {
+    list: assert_equal_sequence,
+    int: assert_equal_value,
+    float: assert_equal_value,
+    str: assert_equal_value,
+    unicode: assert_equal_value, #sequence,
+    dict: assert_equal_dict,
+    set: assert_equal_set,
+}
+
+def assert_equal(expected, actual):
+    assert_same_type(expected, actual)
+    compare_proc = _compare_procs.get(type(expected), assert_equal_value)
+    compare_proc(
+        expected,
+        actual
+    )

=== added file 'modules/test_utils/clear_table.py'
--- modules/test_utils/clear_table.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/clear_table.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,4 @@
+
+def clear_table(db, db_table):
+    db(db_table.id).delete()
+    db.commit()

=== added file 'modules/test_utils/find_JSON_format_data_structure.py'
--- modules/test_utils/find_JSON_format_data_structure.py	1970-01-01 00:00:00 +0000
+++ modules/test_utils/find_JSON_format_data_structure.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,54 @@
+
+import re
+from json.decoder import JSONDecoder
+
+__all__ = (
+    "not_found",
+    "cannot_parse_JSON",
+    "find_JSON_format_data_structure"
+)
+
+def not_found(name, string):
+    raise Exception(
+        u"Cannot find %s in %s" % (name, string)
+    )
+    
+def cannot_parse_JSON(string):
+    raise Exception(
+        u"Cannot parse JSON: '%s'" % string
+    )
+
+def find_JSON_format_data_structure(
+    string,
+    name,
+    found,
+    not_found,
+    cannot_parse_JSON
+):
+    """Finds a named JSON-format data structure in the string.
+    
+    The name can be any string.
+    The pattern "name = " will be looked for in the string,
+        and the data structure following it parsed and returned as a python 
+        data structure. 
+    """
+    try:
+        name_start = string.index(name)
+    except ValueError:
+        not_found(name, string)
+    else:
+        name_length = len(name)
+        name_end = name_start + name_length
+        
+        _, remaining = re.Scanner([
+            (r"\s*=\s*", lambda scanner, token: None)
+        ]).scan(
+            string[name_end:]
+        )
+        
+        try:
+            data, end_position = JSONDecoder().raw_decode(remaining)
+        except ValueError, value_error:
+            cannot_parse_JSON(remaining)
+        else:
+            found(data)

=== modified file 'modules/test_utils/run.py'
--- modules/test_utils/run.py	2011-08-11 19:25:52 +0000
+++ modules/test_utils/run.py	2011-09-06 11:51:25 +0000
@@ -1,5 +1,8 @@
 #!python
 
+# capture web2py environment before doing anything else
+
+web2py_environment = dict(globals())
 
 __doc__ = """This script is run from the nose command in the 
 application being tested:
@@ -8,249 +11,81 @@
 
 python2.6 ./applications/eden/tests/nose.py <nose arguments>
 
+web2py runs a file which:
+1. Sets up a plugin. This plugin registers itself so nose can use it.
+2. Runs nose programmatically giving it the plugin
+nose loads the tests via the plugin. 
+when the plugin loads the tests, it injects the web2py environment.
+
 """
 
-########################################
-#
-# web2py runs a file which:
-# 1. Sets up a plugin. This plugin registers itself so nose can use it.
-# 2. Runs nose programmatically giving it the plugin
-#
-# nose loads the tests via the plugin. 
-# when the plugin loads the tests, it injects the web2py environment.
-#
-########################################
-
-
-# @ToDo:
-# in particular, haven't checked that db is being replaced with the test_db
-# (work in progress)
-
-# using --import_models is OK for a single test, but for running a suite,
-# we probably need the test_env function to set up an environment
-
-# @ToDo: Provide an ignore_warns mode so that we can tackle ERRORs 1st
-# but FAILs often give us clues that help us fix ERRORs
-# fixing an error might itself cause a failure, but not be spotted until later.
-
-def use_test_db(db):
-    print "Creating test database..."
-    try:
-        test_db = use_test_db.db
-    except AttributeError:
-        # create test database by copying db
-        test_db_name = "sqlite://testing.sqlite"
-        print "Copying db tables into test database..."
-        test_db = DAL(test_db_name) # Name and location of the test DB file
-        # Copy tables!
-        for tablename in db.tables:
-            table_copy = []
-            for data in db[tablename]:
-                table_copy.append(
-                    copy.copy(data)
-                )
-            test_db.define_table(tablename, *table_copy)
-        use_test_db.db = test_db
-    return test_db
-
+import sys
+
+from types import ModuleType
 import nose
-
+import glob
+import os.path
+import os
 import copy
-def test_env(
-    imports,
-    application,
-    controller,
-    function,
-    folder,
-    globals,
-    create_test_db = use_test_db,
-    _module_root = "applications",
-):
-    """Sets up the unit-testing environment.
-    
-    This involves loading modules as if by web2py.
-    Also we must make a test database.
-    
-    """
-    from gluon.globals import Request
-    globals["Request"] = Request
-    request = Request()
-    request.application = application
-    request.controller = controller
-    request.function = function
-    request.folder = folder
-    
-    globals["request"] = request
-
-    test_db = db#create_test_db(db)
-    
+from gluon.globals import Request
+import unittest
+
+
+def load_module(application_relative_module_path):
     import os
-    for import_path, names in imports:
-        #print import_path
-        #module = {"db": test_db}
-        #module.update(globals)
-        module = dict(globals)
-        path_components = [_module_root] + import_path.split(".")
-        file_path = os.path.join(*path_components)+".py"
-        # execfile is used because it doesn't create a module
-        # and doesn't load the module if it exists.
-        execfile(file_path, module)
-        if names is "*":
-            globals.update(module)
-        else:
-            for name in names:
-                globals[name] = module[name]
-            
-# -----------------------
-
-import os.path
+    web2py_relative_module_path = ".".join((
+        "applications", request.application, application_relative_module_path
+    ))
+    imported_module = __import__(web2py_relative_module_path)
+    for step in web2py_relative_module_path.split(".")[1:]:
+        imported_module = getattr(imported_module, step)
+    return imported_module
+
+web2py_environment["load_module"] = load_module
+web2py_env_module = ModuleType("web2py_env")
+web2py_env_module.__dict__.update(web2py_environment)
+sys.modules["web2py_env"] = web2py_env_module
+
 
 application_name = request.application
-
-model_files_pattern = os.path.join("applications",application_name,"models","*.py")
-import glob
-
-test_env(
-    globals = globals(),
-    application = application_name,
-    controller = "controller",
-    function = "function",
-    folder = "folder",
-    imports = [
-        (application_name+".models.%s" % (module_name[len(model_files_pattern)-4:-3]), "*") 
-            for module_name in glob.glob(model_files_pattern)
-    ]
-)
-
-import sys
-log = sys.stderr.write
-
-import unittest
-from itertools import imap
+application_folder_path = os.path.join("applications",application_name)
+
+application = application_name
+controller = "controller"
+function = "function"
+folder = os.path.join(os.getcwd(), "applications", application_name)
+
+web2py_environment["Request"] = Request
+request = Request()
+request.application = application
+request.controller = controller
+request.function = function
+request.folder = folder
+
+web2py_environment["request"] = request
+current.request = request
+
+controller_configuration = OrderedDict()
+for controller_name in ["default"]+glob.glob(
+    os.path.join(application_folder_path, "controllers", "*.py")
+):
+    controller_configuration[controller_name] = Storage(
+        name_nice = controller_name,
+        description = controller_name,
+        restricted = False,
+        module_type = 0
+    )
+    
+current.deployment_settings.modules = controller_configuration
 
 test_folders = set()
-
-class Web2pyNosePlugin(nose.plugins.base.Plugin):
-    # see: http://somethingaboutorange.com/mrl/projects/nose/0.11.1/plugins/writing.html
-    
-    """This plugin is designed to give the web2py environment to the tests.
-    """
-    score = 0
-    # always enable as this plugin can only 
-    # be selected by running this script
-    enabled = True
-    
-    def __init__(
-        self,
-        application_name,
-        environment,
-        create_test_db,
-        directory_pattern
-    ):
-        super(Web2pyNosePlugin, self).__init__()
-        self.application_name = application_name
-        self.environment = dict(
-            db = db#create_test_db(db)
-        )
-        self.environment.update(environment)
-        self.directory_pattern = directory_pattern
-        
-    def options(self, parser, env):
-        """Register command line options"""
-        return
-        parser.add_option(
-            "--web2py",
-            dest="web2py",
-            action="append",
-            metavar="ATTR",
-            help="Use web2py environment when loading tests"
-        )
-        
-    def wantDirectory(self, dirname):
-        return bool(re.search(self.directory_pattern, dirname))
-
-    def wantFile(self, file_name):
-        return file_name.endswith(".py") and any(
-            imap(file_name.__contains__, test_folders)
-        )
-
-    def wantModule(self, module):
-        return False
-
-    def loadTestsFromName(self, file_name, discovered):
-        """Sets up the unit-testing environment.
-        
-        This involves loading modules as if by web2py.
-        Also we must have a test database.
-        
-        If testing controllers, tests need to set up the request themselves.
-                
-        """
-        if file_name.endswith(".py"):
-#            log(file_name)
-            
-    #        assert 0, file_name
-    #        stop
-
-            # Is it possible that the module could load 
-            # other code that is using the original db?
-            
-            test_globals = self.environment
-
-            # execfile is used because it doesn't create a module
-            # and doesn't load the module into sys.modules if it exists.
-            module_globals = dict(self.environment)
-            execfile(file_name, module_globals)
-
-            import inspect
-            # we have to return something, otherwise nose
-            # will let others have a go, and they won't pass
-            # in the web2py environment, so we'll get errors
-            tests = []
-    
-            for name, thing in module_globals.iteritems():
-                if (
-                    # don't bother with globally imported things 
-                    name not in test_globals \
-                    # unless they have been overridden
-                    or test_globals[name] is not thing
-                ):
-                    if (
-                        isinstance(thing, type)
-                        and issubclass(thing, unittest.TestCase)
-                    ):
-                        # look for test methods
-                        for member_name in dir(thing):
-                            if member_name.startswith("test"):
-                                if callable(getattr(thing, member_name)):
-                                    tests.append(thing(member_name))
-                    elif (
-                        name.startswith("test") 
-                        or name.startswith("Test")
-                    ):
-                        if inspect.isfunction(thing):
-                            function = thing
-                            function_name = name
-                            # things coming from execfile have no module
-                            #print file_name, function_name, function.__module__
-                            if function.__module__ is None:
-                                tests.append(
-                                    nose.case.FunctionTestCase(function)
-                                )
-            return tests
-        else:
-            return
-        
-import re
-
-argv = [
-    #"--verbosity=2",
-    #"--debug=nose"
-]
+argv = []
 
 # folder in which tests are kept
 # non-option arguments (test paths) are made relative to this
-test_root = os.path.join("applications", application_name, "tests", "unit_tests")
+test_root = os.path.join(application_folder_path, "tests", "unit_tests")
+
+current_working_directory = os.getcwd()
 
 disallowed_options = {}
 disallowed_options["-w"] = disallowed_options["--where"] = (
@@ -275,10 +110,10 @@
             argv.append(arg)
     else:
         test_path = arg
-        test_fuller_path = os.path.join(test_root, test_path)
-        test_folders.add(test_fuller_path)
-        if not os.path.exists(test_fuller_path):
-            print "\n", test_fuller_path, "not found"
+        test_folder_fuller_path = os.path.join(test_root, test_path)
+        test_folders.add(test_folder_fuller_path)
+        if not os.path.exists(test_folder_fuller_path):
+            print "\n", test_folder_fuller_path, "not found"
             #sys.exit(1)
 
 # test paths in command line aren't passed, just added to test_folders
@@ -293,14 +128,15 @@
 
 sys.argv[1:] = argv
 
+test_utils = local_import("test_utils")
+
 nose.main(
 # seems at least this version of nose ignores passed in argv
 #    argv = argv, 
     addplugins = nose.plugins.PluginManager([
-        Web2pyNosePlugin(
+        test_utils.Web2pyNosePlugin(
             application_name,
-            globals(),
-            use_test_db,
+            web2py_environment,
             re.compile(
                 re.escape(os.path.sep).join(
                     (
@@ -311,7 +147,8 @@
                         "[^","]*)*)?)?)?$"
                     )
                 )
-            )
+            ),
+            test_folders
         )
     ])
 )

=== added file 'private/prepopulate/default/tasks.cfg'
--- private/prepopulate/default/tasks.cfg	1970-01-01 00:00:00 +0000
+++ private/prepopulate/default/tasks.cfg	2011-09-06 11:51:25 +0000
@@ -0,0 +1,18 @@
+##########################################################################
+# Add a list of csv file to import into the system
+# the list of import file sis a comma separated list as follows:
+# "prefix","tablename","csv file name","stylesheet"
+#
+# The csv file is assumed to be in the same directory as this file
+# The style sheet is assumed to be in either of the following directories:
+#     static/format/s3csv/"prefix"/
+#     static/format/s3csv/
+#
+# For details on how to import data into the system see the following:
+#     zzz_1st_run
+#     s3Tools::S3BulkImporter
+##########################################################################
+"supply","catalog_item","DefaultItems.csv","supply_items.xsl"
+"supply","catalog_item","StandardItems.csv","supply_items.xsl"
+"hrm","skill","DefaultSkillList.csv","skill.xsl"
+"hrm","competency_rating",DefaultSkillCompetency.csv,competency_rating.xsl
\ No newline at end of file

=== removed file 'private/prepopulate/default/tasks.cfg'
--- private/prepopulate/default/tasks.cfg	2011-08-17 15:13:43 +0000
+++ private/prepopulate/default/tasks.cfg	1970-01-01 00:00:00 +0000
@@ -1,18 +0,0 @@
-##########################################################################
-# Add a list of csv file to import into the system
-# the list of import file sis a comma separated list as follows:
-# "prefix","tablename","csv file name","stylesheet"
-#
-# The csv file is assumed to be in the same directory as this file
-# The style sheet is assumed to be in either of the following directories:
-#     static/format/s3csv/"prefix"/
-#     static/format/s3csv/
-#
-# For details on how to import data into the system see the following:
-#     zzz_1st_run
-#     s3Tools::S3BulkImporter
-##########################################################################
-"supply","catalog_item","DefaultItems.csv","supply_items.xsl"
-"supply","catalog_item","StandardItems.csv","supply_items.xsl"
-"hrm","skill","DefaultSkillList.csv","skill.xsl"
-"hrm","competency_rating",DefaultSkillCompetency.csv,competency_rating.xsl

=== modified file 'static/scripts/S3/s3.gis.climate.js'
--- static/scripts/S3/s3.gis.climate.js	2011-06-15 09:47:54 +0000
+++ static/scripts/S3/s3.gis.climate.js	2011-09-06 11:51:25 +0000
@@ -9,154 +9,372 @@
     }
 }
 
-
 ClimateDataMapPlugin = function (config) {
-    var self = this // so no this-clobbering
-    self.data_type_option_names = config.data_type_option_names
-    self.parameter_names = config.parameter_names
-    self.projected_option_type_names = config.projected_option_type_names
-    self.year_min = config.year_min 
-    self.year_max = config.year_max
-
-    self.data_type_label = config.data_type_label
-    self.projected_option_type_label = config.projected_option_type_label
-
-    self.setup = function () {        
-        var graphic = new OpenLayers.Layer.Image(
-            'Test Data',
-            '/eden/climate/climate_image_overlay',
-            new OpenLayers.Bounds(8900000, 3020000, 9850000, 3580000),
-//            new OpenLayers.Bounds(-180, -88.759, 180, 88.759),
-            new OpenLayers.Size(249, 139),
-            {
-            //    numZoomLevels: 3,
-                isBaseLayer:false,
-                opacity: 0.5,
-                transparent:true
-            }
-        );
-        graphic.events.on({
-            loadstart: function() {
-                OpenLayers.Console.log("loadstart");
-            },
-            loadend: function() {
-                OpenLayers.Console.log("loadend");
-            }
-        });
-        map.addLayer(graphic);
+    var plugin = this // let's be explicit!
+    plugin.data_type_option_names = config.data_type_option_names
+    plugin.parameter_names = config.parameter_names
+    plugin.year_min = config.year_min 
+    plugin.year_max = config.year_max
+
+    plugin.data_type_label = config.data_type_label
+    plugin.overlay_data_URL = config.overlay_data_URL
+    plugin.chart_URL = config.chart_URL
+    delete config
+
+    plugin.setup = function () {
+        var overlay_layer = plugin.overlay_layer = new OpenLayers.Layer.Vector(
+            'Climate data map overlay',
+            {
+                isBaseLayer:false,                                
+            }
+        );
+        map.addLayer(overlay_layer);
+
+        // selection
+        OpenLayers.Feature.Vector.style['default']['strokeWidth'] = '2'
+        var selectCtrl = new OpenLayers.Control.SelectFeature(
+            overlay_layer,
+            {
+                clickout: true,
+                toggle: false,
+                multiple: false,
+                hover: false,
+                toggleKey: 'altKey',
+                multipleKey: 'shiftKey',
+                box: true,
+                onSelect: function (feature) {
+                    feature.style.strokeColor = 'black'
+                    feature.style.strokeDashstyle = 'dash'
+                    overlay_layer.drawFeature(feature)
+                },
+                onUnselect: function (feature) {
+                    feature.style.strokeColor = 'none'
+                    overlay_layer.drawFeature(feature)
+                },
+            }
+        );
+        
+        map.addControl(selectCtrl);
+
+        selectCtrl.activate();
     }
-    self.addToMapWindow = function (items) {
-        function toggle_projected_options() {
-            $('#projected-options').toggle(
-                $('#id_Projected').attr('checked') == 'checked'
+    plugin.addToMapWindow = function (items) {
+        var combo_box_size = {
+            width: 120,
+            heigth:25
+        }
+
+        function make_combo_box(
+            data,
+            fieldLabel,
+            hiddenName
+        ) {
+            var options = []
+            each(
+                data,
+                function (option) {
+                    options.push([option, option])
+                }
             )
-        }
-        var climate_data_type_options = [];
-        each(
-            self.data_type_option_names,
-            function (option_name) {
-                var radio_button = new Ext.form.Radio({
-                    name: "data-type",
-                    id: "id_%s" % option_name,
-                    boxLabel: option_name,
-                    checked: option_name == self.data_type_option_names[0],
-                })
-                radio_button.on({
-                    change: toggle_projected_options
-                })
-                climate_data_type_options.push(radio_button)
-            }
-        )
-        var projected_options = [];
-        each(
-            self.projected_option_type_names,
-            function (projected_option_type_name) {
-                projected_options.push(
-                    new Ext.form.Radio({
-                        name: "projected-option-type",
-                        id: "id_%s" % projected_option_type_name,
-                        boxLabel: projected_option_type_name,
-                    })
-                )
-            }
-        )
-        var projected_options_widget = new Ext.form.FieldSet({
-            title: self.projected_option_type_label,
-            items: [
-                new Ext.form.CheckboxGroup({
-                    items: projected_options,
-                    xtype: 'checkboxgroup',
-                    columns: 1
-                })
-            ]
-        })
-
-        var climate_data_type_options = new Ext.form.FieldSet({
-            title: self.data_type_label,
-            items: [
-                new Ext.form.RadioGroup({
-                    items: climate_data_type_options,
-                    columns: 1,
-                })
-            ]
-        })
-        
-        var parameter_options = [];
-        each(
-            self.parameter_names,
-            function (parameter_name) {
-                var checkbox = new Ext.form.Checkbox({
-                    name: parameter_name,
-                    id: "id_%s" % parameter_name,
-                    boxLabel: parameter_name,
-                })
-                parameter_options.push(checkbox)
-            }
-        )
-
-        var parameters_widget = new Ext.form.FieldSet({
-            title: "Parameters",
-            items: [
-                new Ext.form.CheckboxGroup({
-                    items: parameter_options,
-                    xtype: 'checkboxgroup',
-                    columns: 1
-                })
-            ]
-        })
-        
-        var period_widget = new Ext.form.FieldSet({
-            title: "Period",
-            items: [
-                new Ext.form.NumberField({
-                    fieldLabel: "From",
-                    minValue: self.year_min,
-                    maxValue: self.year_max,
-                    value: self.year_min
+            var combo_box = new Ext.form.ComboBox({
+                fieldLabel: fieldLabel,
+                hiddenName: hiddenName,
+                store: new Ext.data.SimpleStore({
+                    fields: ['name', 'option'],
+                    data: options
                 }),
-                new Ext.form.NumberField({
-                    fieldLabel: "To",
-                    minValue: self.year_min,
-                    maxValue: self.year_max,
-                    value: self.year_max
-                })
-            ]
-        })
+                displayField: 'name',
+                typeAhead: true,
+                mode: 'local',
+                triggerAction: 'all',
+                emptyText:'Choose...',
+                selectOnFocus:true
+            })
+            combo_box.setSize(combo_box_size)
+            return combo_box
+        }
+        var data_type_combo_box = make_combo_box(
+            plugin.data_type_option_names,
+            'Data type',
+            'data_type'
+        )
+
+        var variable_combo_box = make_combo_box(
+            plugin.parameter_names,
+            'Variable',
+            'parameter'
+        )
+        
+        var statistic_combo_box = make_combo_box(
+            ['Minimum','Maximum','Average'],
+            'Aggregate values',
+            'statistic'
+        )
+                        
         var climate_data_panel = new Ext.FormPanel({
             id: 'climate_data_panel',
-            title: 'Climate data',
+            title: 'Climate data map overlay',
             collapsible: true,
             collapseMode: 'mini',
             items: [{
                 region: 'center',
                 items: [
-                    climate_data_type_options,
-                    projected_options_widget,
-                    parameters_widget,
-                    period_widget
+                    new Ext.form.FieldSet({
+                        title: 'Data set',
+                        items: [
+                            data_type_combo_box,
+                            variable_combo_box
+                        ]
+                    }),
+                    new Ext.form.FieldSet({
+                        title: 'Period',
+                        items: [
+                            new Ext.form.NumberField({
+                                fieldLabel: 'From',
+                                name: 'from_date',
+                                minValue: plugin.year_min,
+                                maxValue: plugin.year_max,
+                                value: plugin.year_min
+                            }),
+                            new Ext.form.NumberField({
+                                fieldLabel: 'To',
+                                name: 'to_date',
+                                minValue: plugin.year_min,
+                                maxValue: plugin.year_max,
+                                value: plugin.year_max,
+                                size: combo_box_size
+                            })
+                        ]
+                    }),
+                    new Ext.form.FieldSet({
+                        title: 'Map overlay colours',
+                        items: [
+                            statistic_combo_box,
+                        ]
+                    })
                 ]
             }]
         });
+                
+        var update_map_layer_button = new Ext.Button({
+            text: 'Update map layer',
+            disabled: true,
+            handler: function() {
+                plugin.overlay_layer.destroyFeatures()
+                
+                // request new features
+                var form_values = climate_data_panel.getForm().getValues()
+                
+                // add new features
+                $.ajax({
+                    url: plugin.overlay_data_URL,
+                    data: {
+                        data_type: form_values.data_type,
+                        statistic: form_values.statistic,
+                        parameter: form_values.parameter,
+                        from_date: form_values.from_date,
+                        to_date: form_values.to_date
+                    },
+                    success: function(feature_data, status_code) {
+                        function Vector(geometry, attributes, style) {
+                            style.strokeColor= 'none'
+                            style.fillOpacity= 0.8
+                            style.strokeWidth = 1
+
+                            return new OpenLayers.Feature.Vector(
+                                geometry, attributes, style
+                            )
+                        }
+                        function Polygon(components) {
+                            return new OpenLayers.Geometry.Polygon(components)
+                        }
+                        function Point(lon, lat) {
+                            var point = new OpenLayers.Geometry.Point(lat, lon)
+                            return point.transform(
+                                S3.gis.proj4326,
+                                S3.gis.projection_current
+                            )
+                        }
+                        function LinearRing(point_list) {
+                            point_list.push(point_list[0])
+                            return new OpenLayers.Geometry.LinearRing(point_list)
+                        }
+                        eval('var data = '+feature_data)
+                        $('#id_key_min_value').html(data.min)
+                        $('#id_key_max_value').html(data.max)
+                        plugin.overlay_layer.addFeatures(data.features)
+                    }
+                });
+            }
+        });
+
+        function enable_update_layer_button_if_form_complete(
+            box, record, index
+        ) {
+            if (
+                !!data_type_combo_box.getValue() &&
+                !!variable_combo_box.getValue() &&
+                !!statistic_combo_box.getValue()
+            ) {
+                update_map_layer_button.enable()
+            }
+        }
+        data_type_combo_box.on(
+            'change',
+            enable_update_layer_button_if_form_complete
+        );
+        variable_combo_box.on(
+            'change',
+            enable_update_layer_button_if_form_complete
+        );
+        statistic_combo_box.on(
+            'change',
+            enable_update_layer_button_if_form_complete
+        );
+        climate_data_panel.addButton(update_map_layer_button)
+        
+        var show_chart_button = new Ext.Button({
+            text: 'Show chart',
+            disabled: true,
+            handler: function() {
+                // create URL
+                var place_ids = []
+                each(
+                    plugin.overlay_layer.selectedFeatures, 
+                    function (feature) {
+                        place_ids.push(feature.data.id)
+                    }
+                )
+                var form_values = climate_data_panel.getForm().getValues(),
+                    data_type = form_values.data_type,
+                    parameter = form_values.parameter,
+                    from_date = form_values.from_date,
+                    to_date = form_values.to_date,
+                    place_ids = place_ids;
+                
+                var spec = JSON.stringify({
+                    data_type: data_type,
+                    parameter: parameter,
+                    from_date: from_date,
+                    to_date: to_date,
+                    place_ids: place_ids
+                })
+                
+                var chart_name = [
+                    data_type, parameter,
+                    'from', from_date,
+                    'to', to_date,
+                    'for', (
+                        place_ids.length < 3?
+                        'places: '+ place_ids:
+                        place_ids.length+' places'
+                    )
+                ].join(' ')
+
+                // get hold of a chart manager instance
+                if (!plugin.chart_window) {
+                    var chart_window = plugin.chart_window = window.open(
+                        'climate/chart_popup.html',
+                        'chart', 
+                        'width=660,height=600,toolbar=0,resizable=0'
+                    )
+                    chart_window.onload = function () {
+                        chart_window.chart_manager = new chart_window.ChartManager(plugin.chart_URL)
+                        chart_window.chart_manager.addChartSpec(spec, chart_name)
+                    }
+                    chart_window.onbeforeunload = function () {
+                        delete plugin.chart_window
+                    }
+                } else {
+                    // some duplication here:
+                    plugin.chart_window. chart_manager.addChartSpec(spec, chart_name)
+                }
+                
+            }
+        });
+        
+        
+        function enable_show_chart_button_if_data_and_variable_selected(
+            box, record, index
+        ) {
+            if (
+                !!data_type_combo_box.getValue() &&
+                !!variable_combo_box.getValue()
+            ) {
+                show_chart_button.enable()
+            }
+        }
+        
+        data_type_combo_box.on(
+            'change',
+            enable_show_chart_button_if_data_and_variable_selected
+        );
+        
+        variable_combo_box.on(
+            'change',
+            enable_show_chart_button_if_data_and_variable_selected
+        );
+
+
+        
+        climate_data_panel.addButton(show_chart_button)
+        
         items.push(climate_data_panel)
+        
+        var key_panel = new Ext.Panel({
+            id: 'key_panel',
+            title: 'Key',
+            collapsible: true,
+            collapseMode: 'mini',
+            items: [
+                {
+                    layout: {
+                        type: 'table',
+                        columns: 3,
+                    },
+                    defaults: {
+                        width: '100%',
+                        height: 20,
+                        style: 'margin: 10px'
+                    },
+                    items: [
+                        {
+                            tag: 'span',
+                            id: 'id_key_min_value',
+                            style: 'margin: 5px; text-align: center;',
+                            border: false,
+                            items: [
+                                {
+                                    html:'Min',
+                                    border: false
+                                }
+                            ]
+                        },
+                        new Ext.BoxComponent({
+                            autoEl: {
+                                tag: 'img',
+                                width: 128,
+                                height: 15,
+                                src: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAAABCAYAAAAW0qa2AAAAc0lEQVQoz42QSw6AIAxEX40JEO9/VlBjxg3gD9TFpG06aTrPJjYlYBsNvCAALvce8JZrVuA6X6Snv+kpdwXBwAlsBiIoAQksV536Mr/te3rxDay4r2iNmAFwBcsdVwdfRagDwbC031M8op5j96L8RVEVYQf3hFgEX0OMvQAAAABJRU5ErkJggg=='
+                            }
+                        }),
+                        {
+                            tag: 'span',
+                            id: 'id_key_max_value',
+                            style: 'margin: 5px; text-align: center',
+                            border: false,
+                            items: [
+                                {
+                                    html:'Max',
+                                    border: false
+                                }
+                            ]
+                        }
+                    ]                
+                }
+            ]
+        })
+        
+        items.push(key_panel)
     }
 }

=== added file 'tests/__init__.py'
=== added directory 'tests/climate'
=== added file 'tests/climate/__init__.py'
--- tests/climate/__init__.py	1970-01-01 00:00:00 +0000
+++ tests/climate/__init__.py	2011-09-06 11:51:25 +0000
@@ -0,0 +1,101 @@
+
+ClimateDataPortal = local_import('ClimateDataPortal')
+
+def clear_tables():
+    ClimateDataPortal.place.truncate()
+    ClimateDataPortal.rainfall_mm.truncate()
+    ClimateDataPortal.temperature_celsius.truncate()
+    db.commit()
+#clear_tables()
+
+def frange(start, end, inc=1.0):
+    value = start
+    i = 0