Merge lp:~graeme-acm/sahana-eden/Exporter into lp:sahana-eden/vita

[Graeme Foster]

[nursix (nursix.org)]

The codec should get the resource as only means of data input, from where it can choose xml, json or plain Rows export as needed. The exporter shall maintain the workflow, which though includes the schema documents.

A good start, but needs elaboration. Can't merge this as it breaks S3Search, please include the necessary changes there.

review: Needs Fixing

Unmerged revisions

2384. By graeme-acm on 2011-09-06

minor bug fix

2383. By graeme-acm on 2011-09-05

First run at exporter codec design

=== modified file 'modules/s3/s3codec.py'
--- modules/s3/s3codec.py	2011-09-07 19:43:37 +0000
+++ modules/s3/s3codec.py	2011-09-08 23:52:25 +0000
@@ -30,7 +30,12 @@
     OTHER DEALINGS IN THE SOFTWARE.
 """
 
+<<<<<<< TREE
 __all__ = ["S3Codec"]
+=======
+__all__ = ["S3Codecs",
+           "S3Codec"]
+>>>>>>> MERGE-SOURCE
 
 import datetime
 try:
@@ -43,8 +48,237 @@
 
 from xml.sax.saxutils import escape, unescape
 from gluon import current
-
-# =============================================================================
+from gluon.contenttype import contenttype
+
+try:
+    from cStringIO import StringIO    # Faster, where available
+except:
+    from StringIO import StringIO
+
+# =============================================================================
+
+class S3Codec(object):
+    @staticmethod
+    def getCodec(representation):
+        codecs = {"csv": S3CSV,
+                  "xls": S3XLS,
+                  "json": S3JSON,
+                 }
+        if representation in codecs.keys():
+            return codecs[representation]()
+        else:
+            return None
+
+    def decode(self, resource, source):
+        """
+            API Method to decode a source into an ElementTree, to be
+            implemented by the subclass
+
+            @param resource: the S3Resource
+            @param source: the source
+
+            @returns: an S3XML ElementTree
+        """
+        raise NotImplementedError
+
+    def encode(self, resource, data, **attr):
+        """
+            API Method to encode data into the target format
+
+            @param data: a dict of database rows
+
+            @returns: a handle to the output
+        """
+        raise NotImplementedError
+
+    def generateFileName(self, resource):
+        servername = current.request and "%s_" % current.request.env.server_name or ""
+        return "%s%s.csv" % (servername, resource.tablename)
+
+# =============================================================================
+class S3CSV(S3Codec):
+
+    def encode(self, resource, data, **attr):
+        """
+            API Method to encode data into the target format.
+            Side effect will set up the correct response headers 
+
+            @param data: a dict of database rows
+
+            @returns: a handle to the output
+        """
+        if current.response:
+            filename = self.generateFileName(resource)
+            current.response.headers["Content-Type"] = contenttype(".csv")
+            current.response.headers["Content-disposition"] = "attachment; filename=%s" % filename
+
+        return str(data["list"])
+
+# =============================================================================
+class S3XLS(S3Codec):
+
+    def encode(self, resource, data, **attr):
+        """
+            API Method to encode data into the target format.
+            Side effect will set up the correct response headers 
+
+            @param data: a dict of database rows
+
+            @returns: a handle to the output
+        """
+        try:
+            import xlwt
+        except ImportError:
+            self.manager.session.error = self.ERROR.XLWT_ERROR
+            redirect(URL(extension=""))
+
+        session = current.session
+        request = current.request
+        response = current.response
+
+        table = resource.table
+        COL_WIDTH_MULTIPLIER = 360
+        ROW_ALTERNATING_COLOURS = [0x2A, 0x2B]
+        rowCnt = 0
+        colCnt = 0
+
+        output = StringIO()
+
+        book = xlwt.Workbook(encoding="utf-8")
+        sheet1 = book.add_sheet(str(table))
+
+        # Find fields
+        fields = data["list_fields"]
+        for field in fields:
+            if field.label == 'Id':
+                fields.remove(field)
+        if not fields:
+            fields = [table.id]
+
+        group_by =  attr.get("report_groupby")
+
+        styleLargeHeader = xlwt.XFStyle()
+        styleLargeHeader.font.bold = True
+        styleLargeHeader.font.height = 400
+        styleLargeHeader.alignment.horz = styleLargeHeader.alignment.HORZ_CENTER
+        styleLargeHeader.pattern.pattern = styleLargeHeader.pattern.SOLID_PATTERN
+        styleLargeHeader.pattern.pattern_fore_colour = 0x2C
+
+        styleHeader = xlwt.XFStyle()
+        styleHeader.font.bold = True
+        styleHeader.num_format_str = "D-MMM-YYYY h:mm"
+        styleHeader.pattern.pattern = styleHeader.pattern.SOLID_PATTERN
+        styleHeader.pattern.pattern_fore_colour = 0x2C
+
+        # Title row
+        currentRow = sheet1.row(rowCnt)
+        totalRows = len(fields)-1
+        if group_by != None:
+            totalRows -= 1
+        sheet1.write_merge(rowCnt, rowCnt, 0, totalRows-1, request.function, styleLargeHeader)
+        currentRow.write(totalRows, request.now, styleHeader)
+        currentRow.height = 440
+        rowCnt += 1
+        currentRow = sheet1.row(rowCnt)
+
+        # Header row
+        fieldWidth=[]
+        for field in fields:
+            if group_by != None:
+                if field.label == group_by.label:
+                    continue
+            currentRow.write(colCnt, str(field.label), styleHeader)
+            width=len(field.label)*COL_WIDTH_MULTIPLIER
+            fieldWidth.append(width)
+            sheet1.col(colCnt).width = width
+            colCnt += 1
+
+        # fix the size of the last column to display the date
+        if 16*COL_WIDTH_MULTIPLIER > width:
+            sheet1.col(totalRows).width = 16*COL_WIDTH_MULTIPLIER
+
+        styleSubHeader = xlwt.XFStyle()
+        styleSubHeader.font.bold = True
+        styleSubHeader.pattern.pattern = styleHeader.pattern.SOLID_PATTERN
+        styleSubHeader.pattern.pattern_fore_colour = 0x18
+        styleOdd = xlwt.XFStyle()
+        styleOdd.pattern.pattern = styleOdd.pattern.SOLID_PATTERN
+        styleOdd.pattern.pattern_fore_colour = ROW_ALTERNATING_COLOURS[0]
+        styleEven = xlwt.XFStyle()
+        styleEven.pattern.pattern = styleEven.pattern.SOLID_PATTERN
+        styleEven.pattern.pattern_fore_colour = ROW_ALTERNATING_COLOURS[1]
+        subheading = None
+        for item in data["list"]:
+            # Item details
+            rowCnt += 1
+            currentRow = sheet1.row(rowCnt)
+            colCnt = 0
+            if rowCnt%2 == 0:
+                style = styleEven
+            else:
+                style = styleOdd
+            for field in fields:
+                represent = resource.manager.represent(field,
+                                                       record=item,
+                                                       strip_markup=True,
+                                                       xml_escape=True)
+                if group_by != None:
+                    if field.label == report_groupby.label:
+                        if subheading != represent:
+                            subheading = represent
+                            sheet1.write_merge(rowCnt, rowCnt, 0, totalRows, represent, styleSubHeader)
+                            rowCnt += 1
+                            currentRow = sheet1.row(rowCnt)
+                            if rowCnt%2 == 0:
+                                style = styleEven
+                            else:
+                                style = styleOdd
+                        continue
+                tab, col = str(field).split(".")
+                # Check for Date formats
+                coltype =field.type
+                if coltype == "date":
+                    style.num_format_str = "D-MMM-YY"
+                elif coltype == "datetime":
+                    style.num_format_str = "M/D/YY h:mm"
+                elif coltype == "time":
+                    style.num_format_str = "h:mm:ss"
+                currentRow.write(colCnt, unicode(represent), style)
+                width = len(unicode(represent))*COL_WIDTH_MULTIPLIER
+                if width > fieldWidth[colCnt]:
+                    fieldWidth[colCnt] = width
+                    sheet1.col(colCnt).width = width
+                colCnt += 1
+        sheet1.panes_frozen = True
+        sheet1.horz_split_pos = 2
+        book.save(output)
+        output.seek(0)
+        filename = self.generateFileName(resource)
+        response.headers["Content-Type"] = contenttype(".xls")
+        response.headers["Content-disposition"] = "attachment; filename=\"%s\"" % filename
+        return output.read()
+
+
+
+# =============================================================================
+class S3JSON(S3Codec):
+
+    def encode(self, resource, data, **attr):
+        """
+            API Method to encode data into the target format.
+            Side effect will set up the correct response headers 
+
+            @param data: a dict of database rows
+
+            @returns: a handle to the output
+        """
+        if current.response:
+            current.response.headers["Content-Type"] = "application/json"
+
+        return data["list"].json()
+# =============================================================================
+
+
 
 class S3Codec(object):
     """
@@ -202,6 +436,7 @@
 
         return "%s}" % output
 
+<<<<<<< TREE
 # =============================================================================
 class S3XLS(S3Codec):
 
@@ -220,4 +455,6 @@
     def noop(self):
         pass
 # =============================================================================
+=======
+>>>>>>> MERGE-SOURCE
 

=== modified file 'modules/s3/s3export.py'
--- modules/s3/s3export.py	2011-08-07 10:24:58 +0000
+++ modules/s3/s3export.py	2011-09-08 23:52:25 +0000
@@ -38,252 +38,45 @@
 
 __all__ = ["S3Exporter"]
 
-import datetime
-
-try:
-    from cStringIO import StringIO    # Faster, where available
-except:
-    from StringIO import StringIO
-
-from gluon import HTTP, redirect, URL, current
-from gluon.storage import Storage
-from gluon.contenttype import contenttype
-
-from lxml import etree
+
+from s3method import S3Method
+from s3codecs import S3Codec
 
 # *****************************************************************************
-class S3Exporter(object):
+class S3Exporter(S3Method):
     """
         Exporter toolkit
     """
 
-    def __init__(self, manager):
-        """
-            Constructor
-
-            @param manager: the S3ResourceController
-
-            @todo 2.3: error message completion
-        """
-
-        self.manager = manager
-        T = current.T
-        self.s3 = self.manager.s3
-
-        self.ERROR = Storage(
-            REPORTLAB_ERROR = T("ReportLab not installed"),
-            GERALDO_ERROR = T("Geraldo not installed"),
-            NO_RECORDS = T("No records in this resource"),
-            XLWT_ERROR = T("Xlwt not installed"),
-        )
-
-    # -------------------------------------------------------------------------
-    def csv(self, resource):
-        """
-            Export resource as CSV
-
-            @param resource: the resource to export
-
-            @note: export does not include components!
-
-            @todo: implement audit
-        """
-
-        db = current.db
-        request = current.request
-        response = current.response
-        tablename = resource.tablename
-        query = resource.get_query()
-
-        if response:
-            servername = request and "%s_" % request.env.server_name or ""
-            filename = "%s%s.csv" % (servername, tablename)
-            response.headers["Content-Type"] = contenttype(".csv")
-            response.headers["Content-disposition"] = "attachment; filename=%s" % filename
-
-        return str(db(query).select())
-
-
-    # -------------------------------------------------------------------------
-    def xls(self, resource, list_fields=None, report_groupby=None):
-        """
-            Export a resource as Microsoft Excel spreadsheet
-
-            @param resource: the resource
-            @param list_fields: fields to include in list views
-            @param report_groupby: a Field object of the field to group the records by
-
-            @note: export does not include components!
-
-            @todo 2.3: PEP-8
-            @todo 2.3: implement audit
-            @todo 2.3: use S3Resource.readable_fields
-            @todo 2.3: use separate export_fields instead of list_fields
-        """
-        try:
-            import xlwt
-        except ImportError:
-            self.manager.session.error = self.ERROR.XLWT_ERROR
-            redirect(URL(extension=""))
-
-
-        db = current.db
-        session = current.session
-        request = current.request
-        response = current.response
-
-        table = resource.table
-        query = resource.get_query()
-        COL_WIDTH_MULTIPLIER = 360
-        ROW_ALTERNATING_COLOURS = [0x2A, 0x2B]
-        rowCnt = 0
-        colCnt = 0
-
-
-
-        output = StringIO()
-
-        if report_groupby != None:
-            items = db(query).select(table.ALL, orderby=report_groupby)
-        else:
-            items = db(query).select(table.ALL)
-
-        book = xlwt.Workbook(encoding="utf-8")
-        sheet1 = book.add_sheet(str(table))
-
-        # Find fields
-        fields = None
-        fields = resource.readable_fields(subset=list_fields)
-        for field in fields:
-            if field.label == 'Id':
-                fields.remove(field)
-        if not fields:
-            fields = [table.id]
-
-        styleLargeHeader = xlwt.XFStyle()
-        styleLargeHeader.font.bold = True
-        styleLargeHeader.font.height = 400
-        styleLargeHeader.alignment.horz = styleLargeHeader.alignment.HORZ_CENTER
-        styleLargeHeader.pattern.pattern = styleLargeHeader.pattern.SOLID_PATTERN
-        styleLargeHeader.pattern.pattern_fore_colour = 0x2C
-
-        styleHeader = xlwt.XFStyle()
-        styleHeader.font.bold = True
-        styleHeader.num_format_str = "D-MMM-YYYY h:mm"
-        styleHeader.pattern.pattern = styleHeader.pattern.SOLID_PATTERN
-        styleHeader.pattern.pattern_fore_colour = 0x2C
-
-        # Title row
-        currentRow = sheet1.row(rowCnt)
-        totalRows = len(fields)-1
-        if report_groupby != None:
-            totalRows -= 1
-        sheet1.write_merge(rowCnt, rowCnt, 0, totalRows-1, request.function, styleLargeHeader)
-        currentRow.write(totalRows, request.now, styleHeader)
-        currentRow.height = 440
-        rowCnt += 1
-        currentRow = sheet1.row(rowCnt)
-
-        # Header row
-        fieldWidth=[]
-        for field in fields:
-            if report_groupby != None:
-                if field.label == report_groupby.label:
-                    continue
-            currentRow.write(colCnt, str(field.label), styleHeader)
-            width=len(field.label)*COL_WIDTH_MULTIPLIER
-            fieldWidth.append(width)
-            sheet1.col(colCnt).width = width
-            colCnt += 1
-
-        # fix the size of the last column to display the date
-        if 16*COL_WIDTH_MULTIPLIER > width:
-            sheet1.col(totalRows).width = 16*COL_WIDTH_MULTIPLIER
-
-        styleSubHeader = xlwt.XFStyle()
-        styleSubHeader.font.bold = True
-        styleSubHeader.pattern.pattern = styleHeader.pattern.SOLID_PATTERN
-        styleSubHeader.pattern.pattern_fore_colour = 0x18
-        styleOdd = xlwt.XFStyle()
-        styleOdd.pattern.pattern = styleOdd.pattern.SOLID_PATTERN
-        styleOdd.pattern.pattern_fore_colour = ROW_ALTERNATING_COLOURS[0]
-        styleEven = xlwt.XFStyle()
-        styleEven.pattern.pattern = styleEven.pattern.SOLID_PATTERN
-        styleEven.pattern.pattern_fore_colour = ROW_ALTERNATING_COLOURS[1]
-        subheading = None
-        for item in items:
-            # Item details
-            rowCnt += 1
-            currentRow = sheet1.row(rowCnt)
-            colCnt = 0
-            if rowCnt%2 == 0:
-                style = styleEven
-            else:
-                style = styleOdd
-            for field in fields:
-                represent = self.manager.represent(field,
-                                                   record=item,
-                                                   strip_markup=True,
-                                                   xml_escape=True)
-                if report_groupby != None:
-                    if field.label == report_groupby.label:
-                        if subheading != represent:
-                            subheading = represent
-                            sheet1.write_merge(rowCnt, rowCnt, 0, totalRows, represent, styleSubHeader)
-                            rowCnt += 1
-                            currentRow = sheet1.row(rowCnt)
-                            if rowCnt%2 == 0:
-                                style = styleEven
-                            else:
-                                style = styleOdd
-                        continue
-                tab, col = str(field).split(".")
-                # Check for Date formats
-                coltype = db[tab][col].type
-                if coltype == "date":
-                    style.num_format_str = "D-MMM-YY"
-                elif coltype == "datetime":
-                    style.num_format_str = "M/D/YY h:mm"
-                elif coltype == "time":
-                    style.num_format_str = "h:mm:ss"
-                currentRow.write(colCnt, unicode(represent), style)
-                width = len(unicode(represent))*COL_WIDTH_MULTIPLIER
-                if width > fieldWidth[colCnt]:
-                    fieldWidth[colCnt] = width
-                    sheet1.col(colCnt).width = width
-                colCnt += 1
-        sheet1.panes_frozen = True
-        sheet1.horz_split_pos = 2
-        book.save(output)
-        output.seek(0)
-        response.headers["Content-Type"] = contenttype(".xls")
-        filename = "%s_%s.xls" % (request.env.server_name, str(table))
-        response.headers["Content-disposition"] = "attachment; filename=\"%s\"" % filename
-        return output.read()
-
-
-    # -------------------------------------------------------------------------
-    def json(self, resource,
-             start=None,
-             limit=None,
-             fields=None,
-             orderby=None):
-        """
-            Export a resource as JSON
-
-            @note: export does not include components!
-
-            @param resource: the resource to export
-            @param start: index of the first record to export (for slicing)
-            @param limit: maximum number of records to export (for slicing)
-            @param fields: fields to include in the export (None for all fields)
-        """
-
-        response = current.response
-
-        if fields is None:
-            fields = [f for f in resource.table if f.readable]
-
+    def apply_method(self, r, **attr):
+        """
+
+            @param r: the S3Request
+            @param attr: dictionary of parameters for the method handler
+
+            @returns: output object to send to the view
+        """
+        resource = self.resource
+        # get values out of attr
+        field_list = attr.get("export_fields")
+        if field_list == None:
+            field_list = attr.get("field_list")
+        if field_list == None:
+            field_list = [f for f in resource.table if f.readable]
+        start = attr.get("start")
+        limit = attr.get("limit")
+        orderby =  attr.get("order_by")
+        groupby =  attr.get("report_groupby")
+
+        if orderby == None and groupby != None:
+            orderby = groupby
+
+        # get the codec representation
+        codec = S3Codec.getCodec(r.representation)
+        if codec == None:
+            r.error(501, self.manager.ERROR.BAD_FORMAT)
+
+        # get the data
         attributes = dict()
 
         if orderby is not None:
@@ -293,14 +86,13 @@
         if limitby is not None:
             attributes.update(limitby=limitby)
 
-        # Get the rows and return as json
-        rows = resource.select(*fields, **attributes)
-
-        if response:
-            response.headers["Content-Type"] = "application/json"
-
-        return rows.json()
+        rows = resource.select(*field_list, **attributes)
+        
+        data = {"list_fields": field_list,
+                "list":rows}
+
+        # pass the data to the codec & return
+        return codec.encode(resource, data, **attr)
 
 
 # *****************************************************************************
-

=== modified file 'modules/s3/s3rest.py'
--- modules/s3/s3rest.py	2011-09-07 20:14:40 +0000
+++ modules/s3/s3rest.py	2011-09-08 23:52:25 +0000
@@ -155,11 +155,19 @@
 
         self.linker = S3RecordLinker(self)
         self.xml = S3XML(self)
+<<<<<<< TREE
         self.exporter = S3Exporter(self)
         self.sync = S3Sync()
 
         # Codecs
         self.codecs = Storage()
+=======
+# Need to modify S3Search which uses exporter
+#        self.exporter = S3Exporter(self)
+#
+#        # Codecs
+#        self.codecs = Storage()
+>>>>>>> MERGE-SOURCE
 
         # Default method handlers (override in config)
         self.crud = S3Method()
@@ -701,6 +709,7 @@
                          http=["GET"], transform=True)
         self.set_handler("options", self.get_options,
                          http=["GET"], transform=True)
+<<<<<<< TREE
         self.set_handler("sync", manager.sync,
                          http=["GET", "PUT", "POST"], transform=True)
 
@@ -708,6 +717,19 @@
                          http=["GET"], transform=True)
         self.set_handler("sync_log", manager.sync.log,
                          http=["GET"], transform=False)
+=======
+        self.set_handler("sync", S3Sync(),
+                         http=["GET", "PUT"], transform=True)
+        self.set_handler(None,
+                         S3Exporter(),
+                         http=["GET"],
+                         representation=["csv",
+                                         "xls",
+                                         "json",
+                                         "test_bad_format",
+                                        ]
+                         )
+>>>>>>> MERGE-SOURCE
 
     # -------------------------------------------------------------------------
     # Method handler configuration
@@ -2326,7 +2348,8 @@
         self.ERROR = manager.ERROR
 
         # Export/Import hooks
-        self.exporter = manager.exporter
+# Removed by Graeme
+#        self.exporter = manager.exporter
         self.xml = manager.xml
 
         # Authorization hooks