Merge lp:~nchohan/appscale/GAE1.4.0-namespaces into lp:appscale

[Navraj Chohan]

GAE 1.4.0
Namespace for DB
Fixed Cassandra issue with improper range queries (showed up for namespaces)
Updated pbserver for putting appid for begin trans responses
Removed appscale versioning from AppServer completely
Fixed path issue for MR streaming to point to hadoop-0.20.2
Moved taskqueue code to new location, no longer in labs (there is an auto function for taskqueue, we should discuss this)
Ran pychecker on a few files, and fixed small code issues

[lp:~nchohan/appscale/GAE1.4.0-namespaces]

628. By Navraj Chohan on 2010-12-24

GAE 1.4.0, Namespace for DB, Fixed Cassandra issue with improper range queries (showed up for namespaces), Updated pbserver for putting appid for begin trans responses, Removed appscale versioning from AppServer completely, Fixed path issue for MR streaming to point to hadoop-0.20.2, Moved taskqueue code to new location, no longer in labs (there is an auto function for taskqueue, we should discuss this), Ran pychecker on a few files, and fixed small code issues

[Chris Bunch (cgb-cs)]

Approved - also added new sample apps to tools

review: Approve

=== modified file 'AppController/helperfunctions.rb'
--- AppController/helperfunctions.rb	2010-12-11 18:01:33 +0000
+++ AppController/helperfunctions.rb	2010-12-24 09:11:16 +0000
@@ -225,10 +225,8 @@
 #             "--address=#{public_ip}",
 #             "--port=#{port}",
 #             "--datastore_path=#{db_location}",
-             "--appscale_version=#{app_version}",
              "/var/apps/#{app_name}/app",
              "-a #{public_ip}",
-             "--appscale_version #{app_version}",
              ">> /var/apps/#{app_name}/log/server.log 2>&1 &"]
       start_app = cmd.join(" ")
       Djinn.log_debug(start_app)
 
=== modified file 'AppDB/appscale_server.py'
--- AppDB/appscale_server.py	2010-12-10 23:34:17 +0000
+++ AppDB/appscale_server.py	2010-12-24 09:11:16 +0000
@@ -127,8 +127,8 @@
 
 
 
-def getTableName(app_id, kind, version):
-  return app_id + "___" + kind + "___" + version
+def getTableName(app_id, kind, namespace):
+  return app_id + "___" + kind + "___" + namespace
 
 def getRowKey(app_id, ancestor_list):
   if ancestor_list == None:
@@ -217,8 +217,8 @@
   return key
 
 
-def getJournalTable(app_id, appscale_version):
-  return JOURNAL_TABLE + "___" + app_id + "___" + str(appscale_version)
+def getJournalTable(app_id, namespace):
+  return JOURNAL_TABLE + "___" + app_id + "___" + namespace
 
 # isChild is None if False
 # if isChild is None, root is ignored 
@@ -296,7 +296,7 @@
  
   # remote api request
   # sends back a response 
-  def remote_request(self, app_id, appscale_version, http_request_data):
+  def remote_request(self, app_id, http_request_data):
     apirequest = remote_api_pb.Request(http_request_data)
     apiresponse = remote_api_pb.Response()
     response = None
@@ -318,35 +318,27 @@
     http_request_data = request_data.contents()
     if method == "Put":
       response, errcode, errdetail = self.put_request(app_id, 
-                                                 appscale_version, 
                                                  http_request_data)
     elif method == "Get":
       response, errcode, errdetail = self.get_request(app_id, 
-                                                 appscale_version, 
                                                  http_request_data)
     elif method == "Delete": 
       response, errcode, errdetail = self.delete_request(app_id, 
-                                                    appscale_version, 
                                                     http_request_data)
     elif method == "RunQuery":
       response, errcode, errdetail = self.run_query(app_id, 
-                                          appscale_version, 
                                           http_request_data)
     elif method == "BeginTransaction":
       response, errcode, errdetail = self.begin_transaction_request(app_id, 
-                                                      appscale_version, 
                                                       http_request_data)
     elif method == "Commit":
       response, errcode, errdetail = self.commit_transaction_request(app_id, 
-                                                      appscale_version,
                                                       http_request_data)
     elif method == "Rollback":
       response, errcode, errdetail = self.rollback_transaction_request(app_id, 
-                                                        appscale_version,
                                                         http_request_data)
     elif method == "AllocateIds":
       response, errcode, errdetail = self.allocate_ids_request(app_id, 
-                                                appscale_version,
                                                 http_request_data)
     elif method == "CreateIndex":
       errcode = datastore_pb.Error.PERMISSION_DENIED
@@ -405,9 +397,10 @@
       print "errdetail:",errdetail
     self.write( apiresponse.Encode() )    
 
-  def run_query(self, app_id, appscale_version, http_request_data):
+  def run_query(self, app_id, http_request_data):
     global app_datastore
     query = datastore_pb.Query(http_request_data)
+    namespace = query.name_space()
     #logger.debug("QUERY:%s" % query)
     results = []
     if query.has_transaction():
@@ -441,7 +434,7 @@
     else:
       kind = query.kind()
       # Fetch query from the datastore # 
-      table_name = getTableName(app_id, kind, appscale_version)
+      table_name = getTableName(app_id, kind, namespace)
       r = app_datastore.get_table( table_name, ENTITY_TABLE_SCHEMA)
       err = r[0]
       if err not in ERROR_CODES: 
@@ -494,7 +487,7 @@
         elif prev_version != long(ii):
           # if the versions don't match, a valid version must be fetched
           journal_key = getJournalKey(row_key, prev_version)
-          journal_table = getJournalTable(app_id, appscale_version)
+          journal_table = getJournalTable(app_id, namespace)
           journal_result = app_datastore.get_entity( journal_table, 
 						     journal_key, 
                                                      JOURNAL_SCHEMA )
@@ -616,14 +609,15 @@
     return (clone_qr_pb.Encode(), 0, "")
 
 
-  def begin_transaction_request(self, app_id, appscale_version, http_request_data):
+  def begin_transaction_request(self, app_id, http_request_data):
     transaction_pb = datastore_pb.Transaction()
     # handle = zk.getTransactionID(app_id)
     handle = zoo_keeper.getTransactionID(app_id)
     transaction_pb.set_handle(handle)
+    transaction_pb.set_app(app_id)
     return (transaction_pb.Encode(), 0, "")
 
-  def commit_transaction_request(self, app_id, appscale_version, http_request_data):
+  def commit_transaction_request(self, app_id, http_request_data):
     txn =  datastore_pb.Transaction(http_request_data)
     commitres_pb = datastore_pb.CommitResponse()
 
@@ -635,13 +629,13 @@
              datastore_pb.Error.INTERNAL_ERROR,
              "Unable to release lock")
 
-  def rollback_transaction_request(self, app_id, appscale_version, http_request_data):
+  def rollback_transaction_request(self, app_id, http_request_data):
     txn =  datastore_pb.Transaction(http_request_data)
     zoo_keeper.notifyFailedTransaction(app_id, txn.handle())
     return (api_base_pb.VoidProto().Encode(), 0, "")
 
 
-  def allocate_ids_request(self, app_id, appscale_version, http_request_data):
+  def allocate_ids_request(self, app_id, http_request_data):
     return (api_base_pb.VoidProto().Encode(),
             datastore_pb.Error.PERMISSION_DENIED,
             'Allocation of block ids not implemented.')
@@ -714,7 +708,6 @@
       rollback_req = datastore_pb.Transaction()
       rollback_req.set_handle(internal_txn)
       self.rollback_transaction_request(app_id, 
-                                        "version", 
                                         rollback_req.Encode())
 
   """ Transaction algorithm for single puts:
@@ -732,7 +725,7 @@
       -Commit the transaction
       -Release the lock from ZK   
   """
-  def put_request(self, app_id, appscale_version, http_request_data):
+  def put_request(self, app_id, http_request_data):
     global app_datastore
     global keySecret
     global tableHashTable
@@ -841,7 +834,6 @@
       if not putreq_pb.has_transaction():
         begintime = time.time()
         txn, err, errcode = self.begin_transaction_request(app_id,
-                                                 appscale_version,
                                                  http_request_data)
         
         # parse from contents
@@ -870,7 +862,8 @@
       # Notify Soap Server of any new tables
       #######################################
       # insert key 
-      table_name = getTableName(app_id, kind, appscale_version)
+      namespace = e.key().name_space()
+      table_name = getTableName(app_id, kind, namespace)
       #print "Put Using table name:",table_name
       # Notify Users/Apps table if a new class is being added 
       if table_name not in tableHashTable:
@@ -879,7 +872,7 @@
         # This function is reentrant
         # If the class was deleted, and added a second time there is no 
         # notifying the users/app server of its creation
-        if tableServer.add_class(app_id, kind, keySecret) == "true":
+        if tableServer.add_class(app_id, kind, namespace, keySecret) == "true":
           tableHashTable[table_name] = 1
         
       # Store One Entity #
@@ -933,7 +926,7 @@
                 "Timeout: Unable to update ZooKeeper on change set for transaction")
       journalPut = putThread()
       journal_key = getJournalKey(row_key, txn.handle())
-      journal_table = getJournalTable(app_id, appscale_version)
+      journal_table = getJournalTable(app_id, namespace)
       journalPut.setup(app_datastore,
                        journal_table,
                        journal_key,
@@ -968,7 +961,6 @@
       if not putreq_pb.has_transaction():
         committime = time.time()
         com_res, errcode, errdetail = self.commit_transaction_request(app_id, 
-                                                                      appscale_version, 
                                                                       txn.Encode())
         if PROFILE: appscale_log.write("COMMIT %d %f\n"%(txn.handle(), time.time() - committime))
         
@@ -982,7 +974,7 @@
     return (putresp_pb.Encode(), 0, "")
 
 
-  def get_request(self, app_id, appscale_version, http_request_data):
+  def get_request(self, app_id, http_request_data):
     global app_datastore
     getreq_pb = datastore_pb.GetRequest(http_request_data)
     #logger.debug("GET_REQUEST: %s" % getreq_pb)
@@ -1018,7 +1010,8 @@
       if last_path.has_type():
         kind = last_path.type()
       #logger.debug("get: %s___%s___%s %s" % (app_id, kind, appscale_version, str(entity_id)))
-      table_name = getTableName(app_id, kind, appscale_version)
+      namespace = key.name_space()
+      table_name = getTableName(app_id, kind, namespace)
       row_key = getRowKey(app_id,key.path().element_list())
       #print "get row key:" + str(row_key)
       #print "table_name:" + str(table_name)
@@ -1048,7 +1041,7 @@
         if prev_version == long(NONEXISTANT_TRANSACTION):
           entity = None
         else:
-          journal_table = getJournalTable(app_id, appscale_version)
+          journal_table = getJournalTable(app_id, namespace)
           journal_key = getJournalKey(row_key, prev_version)
           r = app_datastore.get_entity(journal_table, journal_key, ENTITY_TABLE_SCHEMA[:1] )  
           err = r[0]
@@ -1075,7 +1068,7 @@
       rollback to know which entity group a possible failed
       transaction belongs to.
   """ 
-  def delete_request(self, app_id, appscale_version, http_request_data):
+  def delete_request(self, app_id, http_request_data):
     global app_datastore
     root_key = None
     txn = None
@@ -1104,14 +1097,13 @@
       last_path = key.path().element_list()[-1]
       if last_path.has_type():
         kind = last_path.type()
-
+      namespace = key.name_space()
       row_key = getRowKey(app_id, key.path().element_list())
 
       # All deletes are transactional and per entity if 
       # not already wrapped in a transaction
       if not delreq_pb.has_transaction():
         txn, err, errcode = self.begin_transaction_request(app_id,
-                                                 appscale_version,
                                                  http_request_data)
         # parse from contents
         txn = datastore_pb.Transaction(txn)
@@ -1140,13 +1132,13 @@
       ##########################
       # Get the previous version
       ########################## 
-      table_name = getTableName(app_id, kind, appscale_version)
+      table_name = getTableName(app_id, kind, namespace)
       field_name_list = ENTITY_TABLE_SCHEMA[1:]
       r = app_datastore.get_entity( table_name, row_key, field_name_list )
       err = r[0]
       if err not in ERROR_CODES: 
         # the table does not exist, hence, the previous value was null
-        # TODO, make its not because the database is down
+        # TODO, make sure its not because the database is down
         r = ["DB_ERROR:", NONEXISTANT_TRANSACTION] #
       if len(r) == 1:
         r.append(NONEXISTANT_TRANSACTION)
@@ -1174,7 +1166,7 @@
                 "Timeout: Unable to update ZooKeeper on change set for transaction")
 
       encoded_delete = DELETED + "/" + row_key 
-      journal_table = getJournalTable(app_id, appscale_version)
+      journal_table = getJournalTable(app_id, namespace)
       journal_key = getJournalKey(row_key, txn.handle())
       
       field_name_list = JOURNAL_SCHEMA
@@ -1191,7 +1183,7 @@
                 datastore_pb.Error.INTERNAL_ERROR,
                 err + ", Unable to write to journal")
  
-      table_name = getTableName(app_id, kind, appscale_version)
+      table_name = getTableName(app_id, kind, namespace)
       field_name_list = ENTITY_TABLE_SCHEMA
       field_value_list =  [encoded_delete, str(txn.handle())]
       err, res = app_datastore.put_entity( table_name, 
@@ -1210,7 +1202,6 @@
  
       if not delreq_pb.has_transaction():
         com_res, errcode, errdetail = self.commit_transaction_request(app_id, 
-                                                                      appscale_version, 
                                                                       txn.Encode())
         if errcode != 0:
           return (delresp_pb.Encode(), errcode, errdetail)
@@ -1218,20 +1209,20 @@
     return (delresp_pb.Encode(), 0, "")
 
 
-  def optimized_delete_request(self, app_id, appscale_version, http_request_data):
+  def optimized_delete_request(self, app_id, http_request_data):
     pass
-  def run_optimized_query(self, app_id, appscale_version, http_request_data):
+  def run_optimized_query(self, app_id, http_request_data):
     return 
-  def optimized_put_request(self, app_id, appscale_version, http_request_data):
+  def optimized_put_request(self, app_id, http_request_data):
     pass
 
-  def void_proto(self, app_id, appscale_version, http_request_data):
+  def void_proto(self, app_id, http_request_data):
     resp_pb = api_base_pb.VoidProto() 
     print "Got void"
     #logger.debug("VOID_RESPONSE: %s to void" % resp_pb)
     return (resp_pb.Encode(), 0, "" )
   
-  def str_proto(self, app_id, appscale_version, http_request_data):
+  def str_proto(self, app_id, http_request_data):
     str_pb = api_base_pb.StringProto( http_request_data )
     composite_pb = datastore_pb.CompositeIndices()
     print "Got a string proto"
@@ -1240,7 +1231,7 @@
     #logger.debug("CompositeIndex response to string: %s" % composite_pb)
     return (composite_pb.Encode(), 0, "" )    
   
-  def int64_proto(self, app_id, appscale_version, http_request_data):
+  def int64_proto(self, app_id, http_request_data):
     int64_pb = api_base_pb.Integer64Proto( http_request_data ) 
     resp_pb = api_base_pb.VoidProto()
     print "Got a int 64"
@@ -1249,7 +1240,7 @@
     #logger.debug("VOID_RESPONSE to int64: %s" % resp_pb)
     return (resp_pb.Encode(), 0, "")
  
-  def compositeindex_proto(self, app_id, appscale_version, http_request_data):
+  def compositeindex_proto(self, app_id, http_request_data):
     compindex_pb = entity_pb.CompositeIndex( http_request_data)
     resp_pb = api_base_pb.VoidProto()
     print "Got Composite Index"
@@ -1300,14 +1291,14 @@
   ##############
   # OTHER TYPE #
   ##############
-  def unknown_request(self, app_id, appscale_version, http_request_data, pb_type):
+  def unknown_request(self, app_id, http_request_data, pb_type):
     #logger.debug("Received Unknown Protocol Buffer %s" % pb_type )
     print "ERROR: Received Unknown Protocol Buffer <" + pb_type +">.",
     print "Nothing has been implemented to handle this Protocol Buffer type."
     print "http request data:"
     print http_request_data 
     print "http done"
-    self.void_proto(app_id, appscale_version, http_request_data)
+    self.void_proto(app_id, http_request_data)
 
   
   #########################
@@ -1322,27 +1313,15 @@
     app_data  = app_data.split(':')
     #logger.debug("POST len: %d" % len(app_data))
 
-    if len(app_data) == 5:
-      app_id, user_email, nick_name, auth_domain, appscale_version = app_data
-      os.environ['AUTH_DOMAIN'] = auth_domain
-      os.environ['USER_EMAIL'] = user_email
-      os.environ['USER_NICKNAME'] = nick_name
-      os.environ['APPLICATION_ID'] = app_id 
-    elif len(app_data) == 4:
+    if len(app_data) == 4:
       app_id, user_email, nick_name, auth_domain = app_data
       os.environ['AUTH_DOMAIN'] = auth_domain
       os.environ['USER_EMAIL'] = user_email
       os.environ['USER_NICKNAME'] = nick_name
       os.environ['APPLICATION_ID'] = app_id 
-      appscale_version = "1"
-    elif len(app_data) == 2:
-      app_id, appscale_version = app_data
-      app_id = app_data[0]
-      os.environ['APPLICATION_ID'] = app_id
     elif len(app_data) == 1:
       app_id = app_data[0]
       os.environ['APPLICATION_ID'] = app_id 
-      appscale_version = "1"
     else:
       #logger.debug("UNABLE TO EXTRACT APPLICATION DATA")
       return
@@ -1352,9 +1331,9 @@
     #logger.debug("For app version: " + appscale_version)
 
     if pb_type == "Request":
-      self.remote_request(app_id, appscale_version, http_request_data)
+      self.remote_request(app_id, http_request_data)
     else:
-      self.unknown_request(app_id, appscale_version, http_request_data, pb_type)
+      self.unknown_request(app_id, http_request_data, pb_type)
     self.finish() 
 def usage():
   print "AppScale Server" 
 
=== modified file 'AppDB/cassandra/py_cassandra.py'
--- AppDB/cassandra/py_cassandra.py	2010-10-04 21:01:14 +0000
+++ AppDB/cassandra/py_cassandra.py	2010-12-24 09:11:16 +0000
@@ -134,8 +134,8 @@
     keyslices = []
     column_parent = ColumnParent(column_family="Standard1")
     predicate = SlicePredicate(column_names=column_names)
-    start_key = table_name
-    end_key = table_name + '~'
+    start_key = table_name + "/"
+    end_key = table_name + '/~'
     try: 
       client = self.__setup_connection()
       keyslices = client.get_range_slice(MAIN_TABLE, 
@@ -211,8 +211,8 @@
     predicate = SlicePredicate(column_names=[])
     curtime = self.timestamp()
     path = ColumnPath(COLUMN_FAMILY)
-    start_key = table_name
-    end_key = table_name + '~'
+    start_key = table_name + "/"
+    end_key = table_name + '/~'
     try: 
       client = self.__setup_connection()
       keyslices = client.get_range_slice(MAIN_TABLE, 
 
=== modified file 'AppDB/dbconstants.py'
--- AppDB/dbconstants.py	2010-05-10 19:52:37 +0000
+++ AppDB/dbconstants.py	2010-12-24 09:11:16 +0000
@@ -1,6 +1,6 @@
 # Constants
 
-import os,sys
+import os
 APPSCALE_HOME=os.environ.get("APPSCALE_HOME")
 
 LOG_DIR = "%s/AppDB/logs" % APPSCALE_HOME
 
=== modified file 'AppDB/helper_functions.py'
--- AppDB/helper_functions.py	2010-06-23 07:11:04 +0000
+++ AppDB/helper_functions.py	2010-12-24 09:11:16 +0000
@@ -52,7 +52,7 @@
    self.loggingOn = True
 
   def debug(self, string):
-    if self.loggingOn == True:
+    if self.loggingOn:
       self.log_logger.info(string)
 
 def randomString(length):
 
=== modified file 'AppDB/hypertable/py_hypertable.py'
--- AppDB/hypertable/py_hypertable.py	2010-11-16 04:25:22 +0000
+++ AppDB/hypertable/py_hypertable.py	2010-12-24 09:11:16 +0000
@@ -76,8 +76,8 @@
     self.lock.acquire()
     self.conn = ThriftClient(self.get_local_ip(), THRIFT_PORT)
     self.ns = self.conn.open_namespace(NS)
-    if PROFILING:
-      self.logger.debug("HT InitConnection: %s"%str(endtime - starttime))
+    #if PROFILING:
+    #  self.logger.debug("HT InitConnection: %s"%str(endtime - starttime))
     return self.conn
 
   def __closeConnection(self, conn):
 
=== modified file 'AppDB/soap_server.py'
--- AppDB/soap_server.py	2010-10-04 21:01:14 +0000
+++ AppDB/soap_server.py	2010-12-24 09:11:16 +0000
@@ -2,28 +2,20 @@
 # 2nd major revision: No longer are tables being cached in memory
 # See LICENSE file
 
-import string, cgi
-import sys
-import os
 
 # we don't use PYTHONPATH now.
 #PYTHON_PATH = os.environ.get("PYTHONPATH")
 #print "Python path: ",PYTHON_PATH
-print sys.path
+#print sys.path
 
+import sys
 import SOAPpy
 import time
-import socket
 import datetime
 import re
-import cgitb; #cgitb.enable()
-import getopt
-import logging
-import logging.handlers
 from dbconstants import *
 import appscale_datastore
 import appscale_logger
-import pickle
 from M2Crypto import SSL
 
 logger = appscale_logger.getLogger("soap_server")
@@ -323,7 +315,7 @@
   if secret != super_secret:
     #logger.error("commit_new_user: bad secret")
     return "Error: bad secret"
-  if DEBUG: "Commiting a new user %s"%user
+  if DEBUG: print "Commiting a new user %s"%user
   error =  "Error: username should be an email"
   # look for the @ and . in the email
   if user.find("@") == -1 or user.find(".") == -1:
@@ -368,7 +360,7 @@
 
   error =  "Error: appname/language can only be alpha numeric"
  
-  if language.isalnum() == False:
+  if not language.isalnum():
     #logger.error("language %s is not alpha numeric" % language)
     if DEBUG: print error
     return error
@@ -411,7 +403,7 @@
     else:
       #logger.error("creating a new app: %s failed %s" % (appname, result[0])) 
       return "false"
-    return "true" 
+    return ret
   else:
     error = "Error: User not found"
     #logger.error(error)
@@ -426,7 +418,6 @@
     #logger.debug("get_tar: bad secret")
     return "Error: bad secret"
   if DEBUG: print "get_tar: entry" 
-  error =  "Error: unable to find application tar ball. "
   result = db.get_entity(APP_TABLE, app_name, ["tar_ball"])
   if result[0] in ERROR_CODES and len(result) == 2:
     #logger.info("get_tar app:%s length of tar %s" % (app_name, str(len(result[1]))) )
@@ -443,7 +434,7 @@
   if DEBUG: print "commit_tar: entry"
 
   #logger.info("commit_tar app:%s, secret:%s" % (app_name, secret))
-  if DEBUG: "Committing a tar for %s"%app_name
+  if DEBUG: print "Committing a tar for %s"%app_name
   if secret != super_secret:
     #logger.error("commit_tar: bad secret")
     return "Error: bad secret"
@@ -625,7 +616,7 @@
     return "false"
   return "true" 
 
-def add_class(appname, classname, secret):
+def add_class(appname, classname, namespace, secret):
   global db
   global super_secret
   global app_schema
@@ -653,7 +644,7 @@
       # already in classes list
       return "true"
 
-  classes += [str(classname)]
+  classes += [str(classname+"___"+namespace)]
   classes = ':'.join(classes)
 
   result = db.put_entity(APP_TABLE, appname, columns, [classes]) 
@@ -673,8 +664,8 @@
   if result[0] not in ERROR_CODES or len(result) == 1:
     #logger.error("delete_app: Unable to get entity for app %s" %appname)
     return "false: unable to get entity for app"
+  """
   owner = result[1]
-  """
   result = db.get_entity(USER_TABLE, owner, ['applications'])
   if result[0] not in ERROR_CODES and len(result) == 1:
     logger.error("delete_app: Unable to get entity for app %s" %appname)
@@ -697,7 +688,7 @@
     return "false: unable to put for user modified app list"
   """
   # look up all the class tables of this app and delete their tables
-  result = db.get_entity(APP_TABLE, appname, ["classes", "version"])
+  result = db.get_entity(APP_TABLE, appname, ["classes"])
   if result[0] not in ERROR_CODES or len(result) == 1:
     #logger.error("delete_app: Unable to get classes for app %s"%appname)
     return "false: unable to get classes for app"
@@ -706,19 +697,13 @@
     classes = result[0].split(':')
   else:
     classes = []
-  if result[1]:
-    appscale_version = result[1]
-  else:
-    appscale_version = "1"
-    #logger.error("delete_app: Unable to get version number for app %s"%appname)
-
   result = db.put_entity(APP_TABLE, appname, ["host", "port"], ["", ""])
   if result[0] not in ERROR_CODES:
     #logger.error("delete_app: Unable to delete instances for app %s"%appname)
     return "false: unable to delete instances"
 
   for classname in classes:
-    table_name = appname + "___" + classname + "___" + appscale_version
+    table_name = appname + "___" + classname
     db.delete_table(table_name)
     #logger.error("delete_app: removed %s"%table_name)
 
@@ -791,8 +776,8 @@
     return "Error: User does not exist" 
 
   result = result[1:]
-  appdrop_rem_token = result[0] 
-  appdrop_rem_token_exp = result[1] 
+  #appdrop_rem_token = result[0] 
+  #appdrop_rem_token_exp = result[1] 
   t = datetime.datetime.now()
   date_change = str(time.mktime(t.timetuple()))
 
 
=== modified file 'AppServer/BUGS' (properties changed: -x to +x)
=== modified file 'AppServer/LICENSE' (properties changed: -x to +x)
=== modified file 'AppServer/README' (properties changed: -x to +x)
=== modified file 'AppServer/RELEASE_NOTES' (properties changed: -x to +x)
--- AppServer/RELEASE_NOTES	2010-11-30 10:37:25 +0000
+++ AppServer/RELEASE_NOTES	2010-12-24 09:11:16 +0000
@@ -3,6 +3,71 @@
 
 App Engine Python SDK - Release Notes
 
+Version 1.4.0
+================================
+- The Always On feature allows applications to pay and keep 3 instances of their
+  application always running, which can significantly reduce application
+  latency.
+- Developers can now enable Warmup Requests. By specifying  a handler in an
+  app's app.yaml, App Engine will attempt to send a Warmup Request to initialize
+  new instances before a user interacts with it. This can reduce the latency an
+  end-user sees for initializing your application.
+- The Channel API is now available for all users.
+- Task Queue has been officially released, and is no longer an experimental
+  feature. The API import paths that use 'labs' have been deprecated. Task queue
+  storage will count towards an application's overall storage quota, and will
+  thus be charged for.
+- The deadline for Task Queue and Cron requests has been raised to 10 minutes.
+  Datastore and API deadlines within those requests remain unchanged.
+- For the Task Queue, developers can specify task retry_parameters in their
+  queue.yaml.
+- Apps that have enabled billing are allowed up to 100 queues with the Task
+  Queue API.
+- Metadata Queries on the datastore for datastore kinds, namespaces, and entity
+  properties are available.
+- URLFetch allowed response size has been increased, up to 32 MB. Request size
+  is still limited to 1 MB.
+- The request and response sizes for the Images API have been increased to
+  32 MB.
+- The total size of Memcache batch operations is increased to 32 MB. The 1 MB
+  limit on individual Memcache objects still applies.
+- The attachment size for outgoing emails has been increased from 1 MB to 10 MB.
+  The size limit for incoming emails is still 10 MB.
+- Size and quantity limits on datastore batch get/put/delete operations have
+  been removed. Individual entities are still limited to 1 MB, but your app may
+  batch as many entities together for get/put/delete calls as the overall
+  datastore deadline will allow for.
+- When iterating over query results, the datastore will now asynchronously
+  prefetch results, reducing latency in many cases by 10-15%.
+- The Admin Console Blacklist page lists the top blacklist rejected visitors.
+- The automatic image thumbnailing service supports arbitrary crop sizes up to
+  1600px.
+- Overall average instance latency in the Admin Console is now a weighted
+  average over QPS per instance.
+- The developer who uploaded an app version can download that version's code
+  using the appcfg.py download_app command. This feature can be disabled on
+  a per application basis in the admin console, under the 'Permissions' tab.
+  Once disabled, code download for the application CANNOT be re-enabled.
+- Fixed an issue where custom Admin Console pages did not work for Google
+  Apps for your Domain users.
+- In the Python runtime, an instance is killed and restarted when a request
+  handler hits DeadlineExceededError. This should fix an issue related to
+  intermittent SystemErrors using Django.
+    http://code.google.com/p/googleappengine/issues/detail?id=772
+- Allow Django initialization to be moved to appengine_config.py to avoid
+  Django version conflicts when mixing webapp.template with pure Django.
+    http://code.google.com/p/googleappengine/issues/detail?id=1758
+- Fixed an issue with OpenId over SSL.
+  http://code.google.com/p/googleappengine/issues/detail?id=3393
+- Fixed an issue on the dev_appserver where login/logout code didn't work using
+  Python 2.6.
+    http://code.google.com/p/googleappengine/issues/detail?id=3566
+- Fixed an issue in the dev_appserver where get_serving_url did not work
+  for transparent, cropped PNGs:
+    http://code.google.com/p/googleappengine/issues/detail?id=3887
+- Fixed an issue with the DatastoreFileStub.
+    http://code.google.com/p/googleappengine/issues/detail?id=3895
+
 Version 1.3.8
 ==================================
 - Builtin app.yaml handlers are available for common application functions,
 
=== modified file 'AppServer/VERSION' (properties changed: -x to +x)
--- AppServer/VERSION	2010-11-30 10:37:25 +0000
+++ AppServer/VERSION	2010-12-24 09:11:16 +0000
@@ -1,3 +1,3 @@
-release: "1.3.8"
-timestamp: 1284157741
+release: "1.4.0"
+timestamp: 1287687253
 api_versions: ['1']
 
=== modified file 'AppServer/demos/guestbook/app.yaml' (properties changed: -x to +x)
=== modified file 'AppServer/google/appengine/api/apiproxy_stub.py'
--- AppServer/google/appengine/api/apiproxy_stub.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/apiproxy_stub.py	2010-12-24 09:11:16 +0000
@@ -27,7 +27,6 @@
 import time
 
 
-
 MAX_REQUEST_SIZE = 1 << 20
 DS_STAT_LEVEL = 31
 logging.addLevelName(DS_STAT_LEVEL, "DS_STAT")
@@ -81,6 +80,7 @@
     messages = []
     assert request.IsInitialized(messages), messages
     start = time.time()
+
     method = getattr(self, '_Dynamic_' + call)
     method(request, response)
     end = time.time()
@@ -88,4 +88,3 @@
     if service == "datastore_v3":
       logging.log(DS_STAT_LEVEL,"qtype %s time %s" % (call, (end-start)))
 
-
 
=== modified file 'AppServer/google/appengine/api/apiproxy_stub_map.py'
--- AppServer/google/appengine/api/apiproxy_stub_map.py	2010-12-10 23:34:17 +0000
+++ AppServer/google/appengine/api/apiproxy_stub_map.py	2010-12-24 09:11:16 +0000
@@ -237,7 +237,6 @@
       service: string
       stub: stub
     """
-    # Changes made to allow changing stubs dynamically 
     #assert not self.__stub_map.has_key(service), repr(service)
     self.__stub_map[service] = stub
 
@@ -370,6 +369,8 @@
     self.__rpc.callback = self.__internal_callback
     self.callback = callback
 
+    self.__class__.__local.may_interrupt_wait = False
+
   def __internal_callback(self):
     """This is the callback set on the low-level RPC object.
 
@@ -589,9 +590,9 @@
       cls.__local.may_interrupt_wait = True
       try:
         running.__rpc.Wait()
-      except apiproxy_errors.InterruptedError:
-        running.__rpc._RPC__exception = None
-        running.__rpc._RPC__traceback = None
+      except apiproxy_errors.InterruptedError, err:
+        err.rpc._RPC__exception = None
+        err.rpc._RPC__traceback = None
     finally:
       cls.__local.may_interrupt_wait = False
     finished, runnning = cls.__check_one(rpcs)
 
=== modified file 'AppServer/google/appengine/api/appinfo.py'
--- AppServer/google/appengine/api/appinfo.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/appinfo.py	2010-12-24 09:11:16 +0000
@@ -34,14 +34,13 @@
 from google.appengine.api import yaml_listener
 from google.appengine.api import yaml_object
 
-
 _URL_REGEX = r'(?!\^)/|\.|(\(.).*(?!\$).'
 _FILES_REGEX = r'(?!\^).*(?!\$).'
 
 _DELTA_REGEX = r'([0-9]+)([DdHhMm]|[sS]?)'
 _EXPIRATION_REGEX = r'\s*(%s)(\s+%s)*\s*' % (_DELTA_REGEX, _DELTA_REGEX)
 
-_SERVICE_RE_STRING = r'(mail|xmpp_message|rest|startup)'
+_SERVICE_RE_STRING = r'(mail|xmpp_message|xmpp_subscribe|xmpp_presence|rest|warmup)'
 
 _PAGE_NAME_REGEX = r'^.+$'
 
 
=== modified file 'AppServer/google/appengine/api/blobstore/__init__.py' (properties changed: -x to +x)
=== modified file 'AppServer/google/appengine/api/blobstore/blobstore_stub.py'
--- AppServer/google/appengine/api/blobstore/blobstore_stub.py	2010-12-13 07:51:16 +0000
+++ AppServer/google/appengine/api/blobstore/blobstore_stub.py	2010-12-24 09:11:16 +0000
@@ -17,8 +17,9 @@
 
 """
 Modifications for AppScale by Navraj Chohan
+
 Datastore backed Blobstore API stub.
-  
+
 Class:
   BlobstoreServiceStub: BlobstoreService stub backed by datastore.
 """
@@ -30,6 +31,7 @@
 
 import os
 import time
+
 from google.appengine.api import apiproxy_stub
 from google.appengine.api import datastore
 from google.appengine.api import datastore_errors
@@ -39,7 +41,6 @@
 from google.appengine.api.blobstore import blobstore_service_pb
 from google.appengine.runtime import apiproxy_errors
 
-import logging
 
 __all__ = ['BlobStorage',
            'BlobstoreServiceStub',
@@ -47,7 +48,6 @@
            'CreateUploadSession',
            'Error',
           ]
-
 BLOB_PORT = "6106"
 
 class Error(Exception):
@@ -84,6 +84,7 @@
                  'success_path': path,
                  'user': user,
                  'state': 'init'})
+
   datastore.Put(entity)
   return str(entity.key())
 
@@ -169,8 +170,6 @@
     self.__time_function = time_function
     self.__next_session_id = 1
     self.__uploader_path = uploader_path
-    self.__block_cache = ""
-    self.__block_key_cache = ""
 
   @property
   def storage(self):
@@ -225,10 +224,7 @@
     """
     session = self._CreateSession(request.success_path(),
                                   users.get_current_user())
-    logging.info("bsstub: %s"%(self.__storage._app_id))
-
     response.set_url('http://%s:%s/%s%s/%s' % (self._GetEnviron('SERVER_NAME'),
-                                            #self._GetEnviron('NGINX_PORT'),
                                             BLOB_PORT,
                                             self.__uploader_path,
                                             self.__storage._app_id,
@@ -245,6 +241,11 @@
       response: Not used but should be a VoidProto.
     """
     for blob_key in request.blob_key_list():
+      key = datastore_types.Key.from_path(blobstore.BLOB_INFO_KIND,
+                                          str(blob_key),
+                                          namespace='')
+
+      datastore.Delete(key)
       self.__storage.DeleteBlob(blob_key)
 
   def _Dynamic_FetchData(self, request, response):
@@ -282,23 +283,14 @@
           blobstore_service_pb.BlobstoreServiceError.BLOB_FETCH_SIZE_TOO_LARGE)
 
     blob_key = request.blob_key()
-    blob_info_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
-                                            blob_key,
-                                            namespace='')
-    try:
-      datastore.Get(blob_info_key)
-    except datastore_errors.EntityNotFoundError, err:
-      raise apiproxy_errors.ApplicationError(
-          blobstore_service_pb.BlobstoreServiceError.BLOB_NOT_FOUND)
-
-    # Find out the block number from the size
-    # Append that key to the info key and fename=tch the data 
-    # Must deal with over lapping boundaries
+    #blob_info_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
+    #                                        blob_key,
+    #                                        namespace='')
     block_count = int(start_index/blobstore.MAX_BLOB_FETCH_SIZE)
     block_modulo = int(start_index%blobstore.MAX_BLOB_FETCH_SIZE)
 
     block_count_end = int(end_index/blobstore.MAX_BLOB_FETCH_SIZE)
-    block_modulo_end = int(end_index%blobstore.MAX_BLOB_FETCH_SIZE)
+    #block_modulo_end = int(end_index%blobstore.MAX_BLOB_FETCH_SIZE)
 
     block_key = str(blob_key) + "__" + str(block_count)
     block_key = datastore.Key.from_path("__BlobChunk__",
@@ -308,12 +300,13 @@
     if self.__block_key_cache != str(block_key):
       try:
         block = datastore.Get(block_key)
-      except datastore_errors.EntityNotFoundError, err:
+      except datastore_errors.EntityNotFoundError:
         raise apiproxy_errors.ApplicationError(
            blobstore_service_pb.BlobstoreServiceError.BLOB_NOT_FOUND)
 
       self.__block_cache = block["block"]
       self.__block_key_cache = str(block_key)
+
     # Matching boundaries, start and end are within one fetch
     if block_count_end == block_count:
       # Is there enough data to satisfy fetch_size bytes?
@@ -337,12 +330,12 @@
                                         namespace='')
     try:
       block = datastore.Get(block_key)
-    except datastore_errors.EntityNotFoundError, err:
+    except datastore_errors.EntityNotFoundError:
       raise apiproxy_errors.ApplicationError(
          blobstore_service_pb.BlobstoreServiceError.BLOB_NOT_FOUND)
 
     self.__block_cache = block["block"]
     self.__block_key_cache = str(block_key)
-    data.append(self.__block_cache[0,fetch_size - data_size]) 
+    data.append(self.__block_cache[0,fetch_size - data_size])
     response.set_data(data)
-    
+ 
 
=== modified file 'AppServer/google/appengine/api/channel/channel.py'
--- AppServer/google/appengine/api/channel/channel.py	2010-11-30 10:40:47 +0000
+++ AppServer/google/appengine/api/channel/channel.py	2010-12-24 09:11:16 +0000
@@ -32,29 +32,24 @@
 from google.appengine.api.channel import channel_service_pb
 from google.appengine.runtime import apiproxy_errors
 
-MAX_DURATION = 60 * 60 * 4
-
-MAX_SIMULTANEOUS_CONNECTIONS = 10
+
+MAXIMUM_CLIENT_ID_LENGTH = 64
+
+MAXIMUM_MESSAGE_LENGTH = 32767
 
 
 class Error(Exception):
   """Base error class for this module."""
 
 
-class InvalidChannelKeyError(Error):
-  """Error that indicates a bad channel id."""
+class InvalidChannelClientIdError(Error):
+  """Error that indicates a bad client id."""
 
-class InvalidChannelKeyError(Error):
-  """Error that indicates a bad channel key."""
 
 class InvalidMessageError(Error):
   """Error that indicates a message is malformed."""
 
 
-class ChannelTimeoutError(Error):
-  """Error that indicates the given channel has timed out."""
-
-
 def _ToChannelError(error):
   """Translate an application error to a channel Error, if possible.
 
@@ -67,11 +62,9 @@
   """
   error_map = {
       channel_service_pb.ChannelServiceError.INVALID_CHANNEL_KEY:
-      InvalidChannelKeyError,
+      InvalidChannelClientIdError,
       channel_service_pb.ChannelServiceError.BAD_MESSAGE:
       InvalidMessageError,
-      channel_service_pb.ChannelServiceError.CHANNEL_TIMEOUT:
-      ChannelTimeoutError
       }
 
   if error.application_error in error_map:
@@ -88,24 +81,52 @@
     return 'xmpp'
 
 
-def create_channel(application_key):
+def _ValidateClientId(client_id):
+  """Valides a client id.
+
+  Args:
+    client_id: The client id provided by the application.
+
+  Returns:
+    If the client id is of type str, returns the original client id.
+    If the client id is of type unicode, returns the id encoded to utf-8.
+
+  Raises:
+    InvalidChannelClientIdError: if client id is not an instance of str or
+        unicode, or if the (utf-8 encoded) string is longer than 64 characters.
+  """
+  if isinstance(client_id, unicode):
+    client_id = client_id.encode('utf-8')
+  elif not isinstance(client_id, str):
+    raise InvalidChannelClientIdError
+
+  if len(client_id) > MAXIMUM_CLIENT_ID_LENGTH:
+    raise InvalidChannelClientIdError
+
+  return client_id
+
+
+def create_channel(client_id):
   """Create a channel.
 
   Args:
-    application_key: A key to identify this channel on the server side.
+    client_id: A string to identify this channel on the server side.
 
   Returns:
-    A string id that the client can use to connect to the channel.
+    A token that the client can use to connect to the channel.
 
   Raises:
-    InvalidChannelTimeoutError: if the specified timeout is invalid.
+    InvalidChannelClientIdError: if clientid is not an instance of str or
+        unicode, or if the (utf-8 encoded) string is longer than 64 characters.
     Other errors returned by _ToChannelError
   """
 
+  client_id = _ValidateClientId(client_id)
+
   request = channel_service_pb.CreateChannelRequest()
   response = channel_service_pb.CreateChannelResponse()
 
-  request.set_application_key(application_key)
+  request.set_application_key(client_id)
 
   try:
     apiproxy_stub_map.MakeSyncCall(_GetService(),
@@ -118,20 +139,33 @@
   return response.client_id()
 
 
-def send_message(application_key, message):
+def send_message(client_id, message):
   """Send a message to a channel.
 
   Args:
-    application_key: The key passed to create_channel.
+    client_id: The client id passed to create_channel.
     message: A string representing the message to send.
 
   Raises:
+    InvalidChannelClientIdError: if client_id is not an instance of str or
+        unicode, or if the (utf-8 encoded) string is longer than 64 characters.
+    InvalidMessageError: if the message isn't a string or is too long.
     Errors returned by _ToChannelError
   """
+  client_id = _ValidateClientId(client_id)
+
+  if isinstance(message, unicode):
+    message = message.encode('utf-8')
+  elif not isinstance(message, str):
+    raise InvalidMessageError
+
+  if len(message) > MAXIMUM_MESSAGE_LENGTH:
+    raise InvalidMessageError
+
   request = channel_service_pb.SendMessageRequest()
   response = api_base_pb.VoidProto()
 
-  request.set_application_key(application_key)
+  request.set_application_key(client_id)
   request.set_message(message)
 
   try:
 
=== modified file 'AppServer/google/appengine/api/channel/channel_service_pb.py'
--- AppServer/google/appengine/api/channel/channel_service_pb.py	2010-11-30 10:40:47 +0000
+++ AppServer/google/appengine/api/channel/channel_service_pb.py	2010-12-24 09:11:16 +0000
@@ -30,14 +30,12 @@
   INTERNAL_ERROR =    1
   INVALID_CHANNEL_KEY =    2
   BAD_MESSAGE  =    3
-  CHANNEL_TIMEOUT =    4
 
   _ErrorCode_NAMES = {
     0: "OK",
     1: "INTERNAL_ERROR",
     2: "INVALID_CHANNEL_KEY",
     3: "BAD_MESSAGE",
-    4: "CHANNEL_TIMEOUT",
   }
 
   def ErrorCode_Name(cls, x): return cls._ErrorCode_NAMES.get(x, "")
 
=== modified file 'AppServer/google/appengine/api/datastore.py'
--- AppServer/google/appengine/api/datastore.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/datastore.py	2010-12-24 09:11:16 +0000
@@ -31,34 +31,26 @@
 
 
 
+
 import heapq
 import itertools
 import logging
 import os
 import re
-import string
 import sys
+import threading
 import traceback
 from xml.sax import saxutils
 
-from google.appengine.api import api_base_pb
-from google.appengine.api import apiproxy_rpc
 from google.appengine.api import apiproxy_stub_map
 from google.appengine.api import capabilities
 from google.appengine.api import datastore_errors
 from google.appengine.api import datastore_types
-from google.appengine.datastore import datastore_index
 from google.appengine.datastore import datastore_pb
-from google.appengine.runtime import apiproxy_errors
+from google.appengine.datastore import datastore_rpc
+from google.appengine.datastore import datastore_query
 from google.appengine.datastore import entity_pb
 
-try:
-  __import__('google.appengine.api.labs.taskqueue.taskqueue_service_pb')
-  taskqueue_service_pb = sys.modules.get(
-      'google.appengine.api.labs.taskqueue.taskqueue_service_pb')
-except ImportError:
-  from google.appengine.api.taskqueue import taskqueue_service_pb
-
 MAX_ALLOWABLE_QUERIES = 30
 
 MAXIMUM_RESULTS = 1000
@@ -72,22 +64,15 @@
 
 _MAX_INDEXED_PROPERTIES = 5000
 
-_MAX_ID_BATCH_SIZE = 1000 * 1000 * 1000
+_MAX_ID_BATCH_SIZE = datastore_rpc._MAX_ID_BATCH_SIZE
 
 Key = datastore_types.Key
 typename = datastore_types.typename
 
-_txes = {}
-
-_ALLOWED_API_KWARGS = frozenset(['rpc'])
-
-_ALLOWED_FAILOVER_READ_METHODS = set(
-    ('Get', 'RunQuery', 'RunCompiledQuery', 'Count', 'Next'))
-
-ARBITRARY_FAILOVER_MS = -1
-
-STRONG_CONSISTENCY = 0
-EVENTUAL_CONSISTENCY = 1
+_ALLOWED_API_KWARGS = frozenset(['rpc', 'config'])
+
+STRONG_CONSISTENCY = datastore_rpc.Configuration.STRONG_CONSISTENCY
+EVENTUAL_CONSISTENCY = datastore_rpc.Configuration.EVENTUAL_CONSISTENCY
 
 _MAX_INT_32 = 2**31-1
 
@@ -161,24 +146,101 @@
   return (keys, multiple)
 
 
-def GetRpcFromKwargs(kwargs):
+def _GetConfigFromKwargs(kwargs):
+  """Get a Configuration object from the keyword arguments.
+
+  This is purely an internal helper for the various public APIs below
+  such as Get().
+
+  Args:
+    kwargs: A dict containing the keyword arguments passed to a public API.
+
+  Returns:
+    A UserRPC instance, or a Configuration instance, or None.
+
+  Raises:
+    TypeError if unexpected keyword arguments are present.
+  """
   if not kwargs:
     return None
   args_diff = set(kwargs) - _ALLOWED_API_KWARGS
   if args_diff:
-    raise TypeError('Invalid arguments: %s' % ', '.join(args_diff))
-  return kwargs.get('rpc')
-
-
-def _MakeSyncCall(service, call, request, response, rpc=None):
+    raise datastore_errors.BadArgumentError(
+      'Unexpected keyword arguments: %s' % ', '.join(args_diff))
+  rpc = kwargs.get('rpc')
+  config = kwargs.get('config')
+  if rpc is not None:
+    if config is not None:
+      raise datastore_errors.BadArgumentError(
+        'Expected rpc= or config= argument but not both')
+    if isinstance(rpc, (apiproxy_stub_map.UserRPC,
+                        datastore_rpc.Configuration)):
+      return rpc
+    raise datastore_errors.BadArgumentError(
+      'rpc= argument should be None or a UserRPC instance')
+  if config is not None:
+    if not isinstance(config, (datastore_rpc.Configuration,
+                               apiproxy_stub_map.UserRPC)):
+      raise datastore_errors.BadArgumentError(
+      'config= argument should be None or a Configuration instance')
+  return config
+
+
+class DatastoreAdapter(datastore_rpc.AbstractAdapter):
+  """Adapter between datatypes defined here (Entity etc.) and protobufs.
+
+  See the base class in datastore_rpc.py for more docs.
+  """
+
+  def key_to_pb(self, key):
+    return key._Key__reference
+
+  def pb_to_key(self, pb):
+    return Key._FromPb(pb)
+
+  def entity_to_pb(self, entity):
+    return entity._ToPb()
+
+  def pb_to_entity(self, pb):
+    return Entity._FromPb(pb)
+
+
+_adapter = DatastoreAdapter()
+_thread_local = threading.local()
+
+_ENV_KEY = '__DATASTORE_CONNECTION_INITIALIZED__'
+
+
+def _GetConnection():
+  """Retrieve a datastore connection local to the thread."""
+  connection = None
+  if os.getenv(_ENV_KEY):
+    try:
+      connection = _thread_local.connection
+    except AttributeError:
+      pass
+  if connection is None:
+    connection = datastore_rpc.Connection(adapter=_adapter)
+    _SetConnection(connection)
+  return connection
+
+
+def _SetConnection(connection):
+  """Sets the datastore connection local to the thread."""
+  _thread_local.connection = connection
+  os.environ[_ENV_KEY] = '1'
+
+
+
+def _MakeSyncCall(service, call, request, response, config=None):
   """The APIProxy entry point for a synchronous API call.
 
   Args:
-    service: string representing which service to call
-    call: string representing which function to call
-    request: protocol buffer for the request
-    response: protocol buffer for the response
-    rpc: datastore.DatastoreRPC to use for this request.
+    service: For backwards compatibility, must be 'datastore_v3'.
+    call: String representing which function to call.
+    request: Protocol buffer for the request.
+    response: Protocol buffer for the response.
+    config: Optional Configuration to use for this request.
 
   Returns:
     Response protocol buffer. Caller should always use returned value
@@ -187,67 +249,105 @@
   Raises:
     apiproxy_errors.Error or a subclass.
   """
-  if not rpc:
-    rpc = CreateRPC(service)
-
-  rpc.make_call(call, request, response)
-  rpc.wait()
-  rpc.check_success()
+  conn = _GetConnection()
+  if isinstance(request, datastore_pb.Query):
+    conn._set_request_read_policy(request, config)
+    conn._set_request_transaction(request)
+  rpc = conn.make_rpc_call(config, call, request, response)
+  conn.check_rpc_success(rpc)
   return response
 
 
-def CreateRPC(service='datastore_v3', deadline=None, callback=None,
-              read_policy=STRONG_CONSISTENCY):
+def CreateRPC(service='datastore_v3',
+              deadline=None, callback=None, read_policy=None):
   """Create an rpc for use in configuring datastore calls.
 
-  Args:
-    deadline: float, deadline for calls in seconds.
-    callback: callable, a callback triggered when this rpc completes,
-      accepts one argument: the returned rpc.
-    read_policy: flag, set to EVENTUAL_CONSISTENCY to enable eventually
-      consistent reads
-
-  Returns:
-    A datastore.DatastoreRPC instance.
-  """
-  return DatastoreRPC(service, deadline, callback, read_policy)
-
-
-class DatastoreRPC(apiproxy_stub_map.UserRPC):
-  """Specialized RPC for the datastore.
-
-  Wraps the default RPC class and sets appropriate values for use by the
-  datastore.
-
-  This class or a sublcass of it is intended to be instatiated by
-  developers interested in setting specific request parameters, such as
-  deadline, on API calls. It will be used to make the actual call.
-  """
-
-  def __init__(self, service='datastore_v3', deadline=None, callback=None,
-               read_policy=STRONG_CONSISTENCY):
-    super(DatastoreRPC, self).__init__(service, deadline, callback)
-    self.read_policy = read_policy
-
-  def make_call(self, call, request, response):
-    if self.read_policy == EVENTUAL_CONSISTENCY:
-      if call not in _ALLOWED_FAILOVER_READ_METHODS:
-        raise datastore_errors.BadRequestError(
-            'read_policy is only supported on read operations.')
-      if call != 'Next':
-        request.set_failover_ms(ARBITRARY_FAILOVER_MS)
-    super(DatastoreRPC, self).make_call(call, request, response)
-
-  def clone(self):
-    """Make a shallow copy of this instance.
-
-    This is usually used when an RPC has been specified with some configuration
-    options and is being used as a template for multiple RPCs outside of a
-    developer's easy control.
-    """
-    assert self.state == apiproxy_rpc.RPC.IDLE
-    return self.__class__(
-        self.service, self.deadline, self.callback, self.read_policy)
+  NOTE: This functions exists for backwards compatibility.  Please use
+  CreateConfig() instead.  NOTE: the latter uses 'on_completion',
+  which is a function taking an argument, wherease CreateRPC uses
+  'callback' which is a function without arguments.
+
+  Args:
+    service: Optional string; for backwards compatibility, must be
+      'datastore_v3'.
+    deadline: Optional int or float, deadline for calls in seconds.
+    callback: Optional callable, a callback triggered when this rpc
+      completes; takes no arguments.
+    read_policy: Optional read policy; set to EVENTUAL_CONSISTENCY to
+      enable eventually consistent reads (i.e. reads that may be
+      satisfied from an older version of the datastore in some cases).
+      The default read policy may have to wait until in-flight
+      transactions are committed.
+
+  Returns:
+    A UserRPC instance.
+  """
+  assert service == 'datastore_v3'
+  conn = _GetConnection()
+  config = None
+  if deadline is not None:
+    config = datastore_rpc.Configuration(deadline=deadline)
+  rpc = conn.create_rpc(config)
+  rpc.callback = callback
+  if read_policy is not None:
+    rpc.read_policy = read_policy
+  return rpc
+
+
+def CreateConfig(**kwds):
+  """Create a Configuration object for use in configuring datastore calls.
+
+  This configuration can be passed to most datastore calls using the
+  'config=...' argument.
+
+  Args:
+    deadline: Optional deadline; default None (which means the
+      system default deadline will be used, typically 5 seconds).
+    on_completion: Optional callback function; default None.  If
+      specified, it will be called with a UserRPC object as argument
+      when an RPC completes.
+    read_policy: Optional read policy; set to EVENTUAL_CONSISTENCY to
+      enable eventually consistent reads (i.e. reads that may be
+      satisfied from an older version of the datastore in some cases).
+      The default read policy may have to wait until in-flight
+      transactions are committed.
+    **kwds: Other keyword arguments as long as they are supported by
+      datastore_rpc.Configuration().
+
+  Returns:
+    A datastore_rpc.Configuration instance.
+  """
+  return datastore_rpc.Configuration(**kwds)
+
+
+def _Rpc2Config(rpc):
+  """Internal helper to construct a Configuration from a UserRPC object.
+
+  If the argument is a UserRPC object, it returns a Configuration
+  object constructed using the same deadline and read_policy;
+  otherwise it returns the argument unchanged.
+
+  NOTE: If the argument is a UserRPC object, its callback is *not*
+  transferred to the Configuration object; the Configuration's
+  on_completion attribute is set to None.  This is done because (a)
+  the signature of on_completion differs from the callback signature;
+  (b) the caller probably doesn't expect the callback to be called
+  more than once; and (c) the callback, being argument-less, wouldn't
+  know which UserRPC object was actually completing.  But yes,
+  technically, this is a backwards incompatibility.
+
+  Args:
+    rpc: None, a UserRPC object, or a datastore_rpc.Configuration object.
+
+  Returns:
+    None or a datastore_rpc.Configuration object.
+  """
+  if rpc is None or isinstance(rpc, datastore_rpc.Configuration):
+    return rpc
+  read_policy = getattr(rpc, 'read_policy', None)
+  return datastore_rpc.Configuration(deadline=rpc.deadline,
+                                     read_policy=read_policy,
+                                     config=_GetConnection().config)
 
 
 def Put(entities, **kwargs):
@@ -261,7 +361,7 @@
 
   Args:
     entities: Entity or list of Entities
-    rpc: datastore.RPC to use for this request.
+    config: Optional Configuration to use for this request.
 
   Returns:
     Key or list of Keys
@@ -269,7 +369,10 @@
   Raises:
     TransactionFailedError, if the Put could not be committed.
   """
-  rpc = GetRpcFromKwargs(kwargs)
+  config = _GetConfigFromKwargs(kwargs)
+  if getattr(config, 'read_policy', None) == EVENTUAL_CONSISTENCY:
+    raise datastore_errors.BadRequestError(
+        'read_policy is only supported on read operations.')
   entities, multiple = NormalizeAndTypeCheck(entities, Entity)
 
   if multiple and not entities:
@@ -280,36 +383,25 @@
       raise datastore_errors.BadRequestError(
           'App and kind must not be empty, in entity: %s' % entity)
 
-  req = datastore_pb.PutRequest()
-  req.entity_list().extend([e._ToPb() for e in entities])
-
-  keys = [e.key() for e in entities]
-  tx = _MaybeSetupTransaction(req, keys)
-
-  try:
-    resp = _MakeSyncCall(
-        'datastore_v3', 'Put', req, datastore_pb.PutResponse(), rpc)
-  except apiproxy_errors.ApplicationError, err:
-    raise _ToDatastoreError(err)
-
-  keys = resp.key_list()
-  num_keys = len(keys)
-  num_entities = len(entities)
-  if num_keys != num_entities:
-    raise datastore_errors.InternalError(
-        'Put accepted %d entities but returned %d keys.' %
-        (num_entities, num_keys))
-
-  for entity, key in zip(entities, keys):
-    entity._Entity__key._Key__reference.CopyFrom(key)
-
-  if tx:
-    tx.entity_group = entities[0].entity_group()
-
-  if multiple:
-    return [Key._FromPb(k) for k in keys]
-  else:
-    return Key._FromPb(resp.key(0))
+  def extra_hook(keys):
+    num_keys = len(keys)
+    num_entities = len(entities)
+    if num_keys != num_entities:
+      raise datastore_errors.InternalError(
+          'Put accepted %d entities but returned %d keys.' %
+          (num_entities, num_keys))
+
+    for entity, key in zip(entities, keys):
+      if entity._Entity__key._Key__reference != key._Key__reference:
+        assert not entity._Entity__key.has_id_or_name()
+        entity._Entity__key._Key__reference.CopyFrom(key._Key__reference)
+
+    if multiple:
+      return keys
+    else:
+      return keys[0]
+
+  return _GetConnection().async_put(config, entities, extra_hook).get_result()
 
 
 def Get(keys, **kwargs):
@@ -329,39 +421,26 @@
   Args:
     # the primary key(s) of the entity(ies) to retrieve
     keys: Key or string or list of Keys or strings
-    rpc: datastore.RPC to use for this request.
+    config: Optional Configuration to use for this request.
 
   Returns:
     Entity or list of Entity objects
   """
-  rpc = GetRpcFromKwargs(kwargs)
+  config = _GetConfigFromKwargs(kwargs)
   keys, multiple = NormalizeAndTypeCheckKeys(keys)
 
   if multiple and not keys:
     return []
-  req = datastore_pb.GetRequest()
-  req.key_list().extend([key._Key__reference for key in keys])
-  _MaybeSetupTransaction(req, keys)
-
-  try:
-    resp = _MakeSyncCall(
-        'datastore_v3', 'Get', req, datastore_pb.GetResponse(), rpc)
-  except apiproxy_errors.ApplicationError, err:
-    raise _ToDatastoreError(err)
-
-  entities = []
-  for group in resp.entity_list():
-    if group.has_entity():
-      entities.append(Entity._FromPb(group.entity()))
+
+  def extra_hook(entities):
+    if multiple:
+      return entities
     else:
-      entities.append(None)
+      if entities[0] is None:
+        raise datastore_errors.EntityNotFoundError()
+      return entities[0]
 
-  if multiple:
-    return entities
-  else:
-    if entities[0] is None:
-      raise datastore_errors.EntityNotFoundError()
-    return entities[0]
+  return _GetConnection().async_get(config, keys, extra_hook).get_result()
 
 
 def Delete(keys, **kwargs):
@@ -374,27 +453,21 @@
   Args:
     # the primary key(s) of the entity(ies) to delete
     keys: Key or string or list of Keys or strings
-    rpc: datastore.RPC to use for this request.
+    config: Optional Configuration to use for this request.
 
   Raises:
     TransactionFailedError, if the Delete could not be committed.
   """
-  rpc = GetRpcFromKwargs(kwargs)
+  config = _GetConfigFromKwargs(kwargs)
+  if getattr(config, 'read_policy', None) == EVENTUAL_CONSISTENCY:
+    raise datastore_errors.BadRequestError(
+        'read_policy is only supported on read operations.')
   keys, multiple = NormalizeAndTypeCheckKeys(keys)
 
   if multiple and not keys:
     return
 
-  req = datastore_pb.DeleteRequest()
-  req.key_list().extend([key._Key__reference for key in keys])
-
-  tx = _MaybeSetupTransaction(req, keys)
-
-  try:
-    _MakeSyncCall(
-        'datastore_v3', 'Delete', req, datastore_pb.DeleteResponse(), rpc)
-  except apiproxy_errors.ApplicationError, err:
-    raise _ToDatastoreError(err)
+  _GetConnection().async_delete(config, keys).get_result()
 
 
 class Entity(dict):
@@ -704,7 +777,7 @@
     return pb
 
   @staticmethod
-  def FromPb(pb):
+  def FromPb(pb, validate_reserved_properties=True):
     """Static factory method. Returns the Entity representation of the
     given protocol buffer (datastore_pb.Entity).
 
@@ -719,10 +792,12 @@
       real_pb.ParseFromString(pb)
       pb = real_pb
 
-    return Entity._FromPb(pb, require_valid_key=False)
+    return Entity._FromPb(
+        pb, require_valid_key=False,
+        validate_reserved_properties=validate_reserved_properties)
 
   @staticmethod
-  def _FromPb(pb, require_valid_key=True):
+  def _FromPb(pb, require_valid_key=True, validate_reserved_properties=True):
     """Static factory method. Returns the Entity representation of the
     given protocol buffer (datastore_pb.Entity). Not intended to be used by
     application developers.
@@ -790,7 +865,8 @@
     for name, value in temporary_values.iteritems():
       decoded_name = unicode(name.decode('utf-8'))
 
-      datastore_types.ValidateReadProperty(decoded_name, value)
+      datastore_types.ValidateReadProperty(
+          decoded_name, value, read_only=(not validate_reserved_properties))
 
       dict.__setitem__(e, decoded_name, value)
 
@@ -874,33 +950,29 @@
   the query. The returned count is cached; successive Count() calls will not
   re-scan the datastore unless the query is changed.
   """
-  ASCENDING = datastore_pb.Query_Order.ASCENDING
-  DESCENDING = datastore_pb.Query_Order.DESCENDING
-
-  ORDER_FIRST = datastore_pb.Query.ORDER_FIRST
-  ANCESTOR_FIRST = datastore_pb.Query.ANCESTOR_FIRST
-  FILTER_FIRST = datastore_pb.Query.FILTER_FIRST
-
-  OPERATORS = {'<':  datastore_pb.Query_Filter.LESS_THAN,
-               '<=': datastore_pb.Query_Filter.LESS_THAN_OR_EQUAL,
-               '>':  datastore_pb.Query_Filter.GREATER_THAN,
-               '>=': datastore_pb.Query_Filter.GREATER_THAN_OR_EQUAL,
-               '=':  datastore_pb.Query_Filter.EQUAL,
-               '==': datastore_pb.Query_Filter.EQUAL,
-               }
-  INEQUALITY_OPERATORS = frozenset(['<', '<=', '>', '>='])
+  ASCENDING = datastore_query.PropertyOrder.ASCENDING
+  DESCENDING = datastore_query.PropertyOrder.DESCENDING
+
+  ORDER_FIRST = datastore_query.QueryOptions.ORDER_FIRST
+  ANCESTOR_FIRST = datastore_query.QueryOptions.ANCESTOR_FIRST
+  FILTER_FIRST = datastore_query.QueryOptions.FILTER_FIRST
+
+  OPERATORS = {'==': datastore_query.PropertyFilter._OPERATORS['=']}
+  OPERATORS.update(datastore_query.PropertyFilter._OPERATORS)
+
+  INEQUALITY_OPERATORS = datastore_query.PropertyFilter._INEQUALITY_OPERATORS
+
   UPPERBOUND_INEQUALITY_OPERATORS = frozenset(['<', '<='])
   FILTER_REGEX = re.compile(
-    '^\s*([^\s]+)(\s+(%s)\s*)?$' % '|'.join(OPERATORS.keys()),
+    '^\s*([^\s]+)(\s+(%s)\s*)?$' % '|'.join(OPERATORS),
     re.IGNORECASE | re.UNICODE)
 
   __kind = None
   __app = None
   __namespace = None
   __orderings = None
-  __cached_count = None
   __hint = None
-  __ancestor = None
+  __ancestor_pb = None
   __compile = None
 
   __cursor = None
@@ -1099,33 +1171,114 @@
       # this query
       Query
     """
-    self.__ancestor = _GetCompleteKeyOrError(ancestor)
+    self.__ancestor_pb = _GetCompleteKeyOrError(ancestor)._ToPb()
     return self
 
   def IsKeysOnly(self):
     """Returns True if this query is keys only, false otherwise."""
     return self.__keys_only
 
-  def GetCompiledCursor(self):
+  def GetQueryOptions(self):
+    """Returns a datastore_query.QueryOptions for the current instance."""
+    return datastore_query.QueryOptions(keys_only=self.__keys_only,
+                                        produce_cursors=self.__compile,
+                                        start_cursor=self.__cursor,
+                                        end_cursor=self.__end_cursor,
+                                        hint=self.__hint)
+
+  def GetQuery(self):
+    """Returns a datastore_query.Query for the current instance."""
+    return datastore_query.Query(app=self.__app,
+                                 namespace=self.__namespace,
+                                 kind=self.__kind,
+                                 ancestor=self.__ancestor_pb,
+                                 filter_predicate=self.GetFilterPredicate(),
+                                 order=self.GetOrder())
+
+  def GetOrder(self):
+    """Gets a datastore_query.Order for the current instance.
+
+    Returns:
+      datastore_query.Order or None if there are no sort orders set on the
+      current Query.
+    """
+
+    orders = [datastore_query.PropertyOrder(property, direction)
+              for property, direction in self.__orderings]
+    if orders:
+      return datastore_query.CompositeOrder(orders)
+    return None
+
+  def GetFilterPredicate(self):
+    """Returns a datastore_query.FilterPredicate for the current instance.
+
+    Returns:
+      datastore_query.FilterPredicate or None if no filters are set on the
+      current Query.
+    """
+    ordered_filters = [(i, f) for f, i in self.__filter_order.iteritems()]
+    ordered_filters.sort()
+
+    property_filters = []
+    for _, filter_str in ordered_filters:
+      if filter_str not in self:
+        continue
+
+      values = self[filter_str]
+      match = self._CheckFilter(filter_str, values)
+      name = match.group(1)
+
+      op = match.group(3)
+      if op is None or op == '==':
+        op = '='
+
+      property_filters.append(datastore_query.make_filter(name, op, values))
+
+    if property_filters:
+      return datastore_query.CompositeFilter(
+          datastore_query.CompositeFilter.AND,
+          property_filters)
+    return None
+
+  def GetCursor(self):
+    """Get the cursor from the last run of this query.
+
+    The source of this cursor varies depending on what the last call was:
+      - Run: A cursor that points immediately after the last result pulled off
+        the returned iterator.
+      - Get: A cursor that points immediately after the last result in the
+        returned list.
+      - Count: A cursor that points immediately after the last result counted.
+
+    Returns:
+      A datastore_query.Cursor object that can be used in subsiquent query
+      requests.
+    """
     try:
-      compiled_cursor = self.__last_iterator.GetCompiledCursor(self)
-      if not compiled_cursor:
+      cursor = self.__cursor_source()
+      if not cursor:
         raise AttributeError()
     except AttributeError:
       raise AssertionError('No cursor available, either this query has not '
                            'been executed or there is no compilation '
                            'available for this kind of query')
-    return compiled_cursor
-
-  def GetCompiledQuery(self):
-    try:
-      if not self.__compiled_query:
-        raise AttributeError()
-    except AttributeError:
-      raise AssertionError('No compiled query available, either this query has '
-                           'not been executed or there is no compilation '
-                           'available for this kind of query')
-    return self.__compiled_query
+    return cursor
+
+  def GetBatcher(self, config=None):
+    """Runs this query and returns a datastore_query.Batcher.
+
+    This is not intended to be used by application developers. Use Get()
+    instead!
+
+    Args:
+      config: Optional Configuration to use for this request.
+
+    Returns:
+      # an iterator that provides access to the query results
+      Iterator
+    """
+    query_options = self.GetQueryOptions().merge(config)
+    return self.GetQuery().run(_GetConnection(), query_options)
 
   def Run(self, **kwargs):
     """Runs this query.
@@ -1142,80 +1295,17 @@
       offset: integer, offset for the query.
       prefetch_count: integer, number of results to return in the first query.
       next_count: number of results to return in subsequent next queries.
-      rpc: datastore.RPC to use for this request.
-
-    Returns:
-      # an iterator that provides access to the query results
-      Iterator
-    """
-    return self._Run(**kwargs)
-
-  def _Run(self, limit=None, offset=None,
-           prefetch_count=None, next_count=None, **kwargs):
-    """Runs this query, with an optional result limit and an optional offset.
-
-    Identical to Run, with the extra optional limit, offset, prefetch_count,
-    next_count parameters. These parameters must be integers >= 0.
-
-    This is not intended to be used by application developers. Use Get()
-    instead!
-
-    Args:
-      limit: integer, limit for the query.
-      offset: integer, offset for the query.
-      prefetch_count: integer, number of results to return in the first query.
-      next_count: number of results to return in subsequent next queries.
-      rpc: datastore.RPC to use for this request.
-
-    Returns:
-      # an iterator that provides access to the query results
-      Iterator
-    """
-    rpc = GetRpcFromKwargs(kwargs)
-    self.__last_iterator, self.__compiled_query = Query._RunInternal(
-        self._ToPb(limit, offset, prefetch_count),
-        next_count=next_count,
-        rpc=rpc)
-
-    return self.__last_iterator
-
-  @staticmethod
-  def _RunInternal(request, next_count=None, rpc=None):
-    """Runs the given request and wraps the result in an iterator.
-
-    Args:
-      request: datastore_pb.query, the request to run.
-      next_count: number of results to return in subsequent next queries.
-      rpc: datastore.RPC to use for this request.
-
-    Returns:
-      (Iterator, datastore_pb.CompiledQuery), the iterator and compiled query
-      that result from running the given request.
-    """
-
-    if rpc:
-      rpc_clone = rpc.clone()
-    else:
-      rpc_clone = None
-
-    try:
-      result = _MakeSyncCall('datastore_v3', 'RunQuery', request,
-                             datastore_pb.QueryResult(), rpc)
-    except apiproxy_errors.ApplicationError, err:
-      try:
-        raise _ToDatastoreError(err)
-      except datastore_errors.NeedIndexError, exc:
-        yaml = datastore_index.IndexYamlForQuery(
-          *datastore_index.CompositeIndexForQuery(request)[1:-1])
-        raise datastore_errors.NeedIndexError(
-          str(exc) + '\nThis query needs this index:\n' + yaml)
-
-    iterator = Iterator(result, query_request_pb=request, batch_size=next_count,
-                        rpc=rpc_clone)
-    if result.has_compiled_query():
-      return iterator, result.compiled_query()
-    else:
-      return iterator, None
+      config: Optional Configuration to use for this request.
+
+    Returns:
+      # an iterator that provides access to the query results
+      Iterator
+    """
+    config = _Rpc2Config(_GetConfigFromKwargs(kwargs))
+    itr = Iterator(self.GetBatcher(config=config))
+    self.__cursor_source = itr.cursor
+    self.__compiled_query_source = itr._compiled_query
+    return itr
 
   def Get(self, limit, offset=0, **kwargs):
     """Fetches and returns a maximum number of results from the query.
@@ -1249,54 +1339,64 @@
       int or long
       # the number of entities to skip
       int or long
-      rpc: datastore.RPC to use for this request.
+      config: Optional Configuration to use for this request. If limit and
+      offset are specified in the config, they are ignored.
 
     Returns:
       # a list of entities
       [Entity, ...]
     """
-    if not isinstance(limit, (int, long)) or limit < 0:
-      raise datastore_errors.BadArgumentError(
-          'Argument to Get named \'limit\' must be an int greater than or '
-          'equal to 0; received %s (a %s)' % (limit, typename(limit)))
-
-    if not isinstance(offset, (int, long)) or offset < 0:
-      raise datastore_errors.BadArgumentError(
-          'Argument to Get named \'offset\' must be an int greater than or '
-          'equal to 0; received %s (a %s)' % (offset, typename(offset)))
-
-    return self._Run(
-        limit=limit, offset=offset, prefetch_count=limit, **kwargs)._Get(limit)
+    config = _Rpc2Config(_GetConfigFromKwargs(kwargs))
+    batcher = self.GetBatcher(datastore_query.QueryOptions(
+        config=config, limit=limit, offset=offset, prefetch_size=limit))
+
+    if limit is None:
+      batch = batcher.next_batch(_MAX_INT_32)
+    else:
+      batch = batcher.next_batch(limit)
+    self.__cursor_source = lambda: batch.end_cursor
+    self.__compiled_query_source = lambda: batch._compiled_query
+    return batch.results
 
   def Count(self, limit=1000, **kwargs):
-    """Returns the number of entities that this query matches. The returned
-    count is cached; successive Count() calls will not re-scan the datastore
-    unless the query is changed.
+    """Returns the number of entities that this query matches.
 
     Args:
       limit, a number or None. If there are more results than this, stop short
       and just return this number. Providing this argument makes the count
       operation more efficient.
-      rpc: datastore.RPC to use for this request.
+      config: Optional Configuration to use for this request.
 
     Returns:
       The number of results.
     """
-    if not self.__cached_count:
-      if limit is None:
-        offset = _MAX_INT_32
-      else:
-        offset = limit
-
-      iterator = self._Run(limit=0, offset=offset, **kwargs)
-      self.__cached_count = iterator._SkippedResults()
-
-    return self.__cached_count
+    if limit is None:
+      offset = _MAX_INT_32
+    else:
+      offset = limit
+
+    config = datastore_query.QueryOptions(
+        config=_Rpc2Config(_GetConfigFromKwargs(kwargs)),
+        limit=0,
+        offset=offset)
+
+    batch = self.GetBatcher(config=config).next()
+    self.__cursor_source = lambda: batch.cursor(0)
+    self.__compiled_query_source = lambda: batch._compiled_query
+    return batch.skipped_results
 
   def __iter__(self):
     raise NotImplementedError(
       'Query objects should not be used as iterators. Call Run() first.')
 
+  def __getstate__(self):
+    state = self.__dict__.copy()
+    if '_Query__cursor_source' in state:
+      del state['_Query__cursor_source']
+    if '_Query__compiled_query_source' in state:
+      del state['_Query__compiled_query_source']
+    return state
+
   def __setitem__(self, filter, value):
     """Implements the [] operator. Used to set filters.
 
@@ -1325,8 +1425,6 @@
       self.__filter_order[filter] = self.__filter_counter
       self.__filter_counter += 1
 
-    self.__cached_count = None
-
   def setdefault(self, filter, value):
     """If the filter exists, returns its value. Otherwise sets it to value.
 
@@ -1336,7 +1434,6 @@
     """
     datastore_types.ValidateProperty(' ', value)
     self._CheckFilter(filter, value)
-    self.__cached_count = None
     return dict.setdefault(self, filter, value)
 
   def __delitem__(self, filter):
@@ -1344,7 +1441,6 @@
     """
     dict.__delitem__(self, filter)
     del self.__filter_order[filter]
-    self.__cached_count = None
 
     match = Query.FILTER_REGEX.match(filter)
     property = match.group(1)
@@ -1447,92 +1543,42 @@
 
     return match
 
+  def _Run(self, limit=None, offset=None,
+           prefetch_count=None, next_count=None, **kwargs):
+    """Deprecated, use .Run instead."""
+    config = _Rpc2Config(_GetConfigFromKwargs(kwargs))
+    return self.Run(config=datastore_query.QueryOptions(
+        config=config,
+        limit=limit,
+        offset=offset,
+        prefetch_size=prefetch_count,
+        batch_size=next_count))
+
   def _ToPb(self, limit=None, offset=None, count=None):
-    """Converts this Query to its protocol buffer representation. Not
-    intended to be used by application developers. Enforced by hiding the
-    datastore_pb classes.
-
-    Args:
-      # an upper bound on the number of results returned by the query.
-      limit: int
-      # number of results that match the query to skip.  limit is applied
-      # after the offset is fulfilled
-      offset: int
-      # the requested initial batch size
-      count: int
-
-    Returns:
-      # the PB representation of this Query
-      datastore_pb.Query
-
-    Raises:
-      BadRequestError if called inside a transaction and the query does not
-      include an ancestor.
+    query_options = datastore_query.QueryOptions(
+        config=self.GetQueryOptions(),
+        limit=limit,
+        offset=offset,
+        batch_size=count)
+    return self.GetQuery()._to_pb(_GetConnection(), query_options)
+
+  def _GetCompiledQuery(self):
+    """Returns the internal-only pb representation of the last query run.
+
+    Do not use.
     """
-
-    if not self.__ancestor and IsInTransaction():
-      raise datastore_errors.BadRequestError(
-        'Only ancestor queries are allowed inside transactions.')
-
-    pb = datastore_pb.Query()
-    _MaybeSetupTransaction(pb, [self.__ancestor])
-
-    if self.__kind is not None:
-      pb.set_kind(self.__kind.encode('utf-8'))
-    pb.set_keys_only(bool(self.__keys_only))
-    if self.__app:
-      pb.set_app(self.__app.encode('utf-8'))
-    datastore_types.SetNamespace(pb, self.__namespace)
-    if self.__compile:
-      pb.set_compile(True)
-    if limit is not None:
-      pb.set_limit(limit)
-    if offset is not None:
-      pb.set_offset(offset)
-    if count is not None:
-      pb.set_count(count)
-    if self.__ancestor:
-      pb.mutable_ancestor().CopyFrom(self.__ancestor._Key__reference)
-
-    if ((self.__hint == self.ORDER_FIRST and self.__orderings) or
-        (self.__hint == self.ANCESTOR_FIRST and self.__ancestor) or
-        (self.__hint == self.FILTER_FIRST and len(self) > 0)):
-      pb.set_hint(self.__hint)
-
-    ordered_filters = [(i, f) for f, i in self.__filter_order.iteritems()]
-    ordered_filters.sort()
-
-    for i, filter_str in ordered_filters:
-      if filter_str not in self:
-        continue
-
-      values = self[filter_str]
-      match = self._CheckFilter(filter_str, values)
-      name = match.group(1)
-
-      props = datastore_types.ToPropertyPb(name, values)
-      if not isinstance(props, list):
-        props = [props]
-
-      op = match.group(3)
-      if op is None:
-        op = '='
-
-      for prop in props:
-        filter = pb.add_filter()
-        filter.set_op(self.OPERATORS[op])
-        filter.add_property().CopyFrom(prop)
-
-    for property, direction in self.__orderings:
-      order = pb.add_order()
-      order.set_property(property.encode('utf-8'))
-      order.set_direction(direction)
-
-    if self.__cursor:
-      pb.mutable_compiled_cursor().CopyFrom(self.__cursor)
-    if self.__end_cursor:
-      pb.mutable_end_compiled_cursor().CopyFrom(self.__end_cursor)
-    return pb
+    try:
+      compiled_query = self.__compiled_query_source()
+      if not compiled_query:
+        raise AttributeError()
+    except AttributeError:
+      raise AssertionError('No compiled query available, either this query has '
+                           'not been executed or there is no compilation '
+                           'available for this kind of query')
+    return compiled_query
+
+  GetCompiledQuery = _GetCompiledQuery
+  GetCompiledCursor = GetCursor
 
 
 def AllocateIds(model_key, size=None, **kwargs):
@@ -1557,48 +1603,24 @@
                in which to allocate IDs
     size: integer, number of IDs to allocate.
     max: integer, upper bound of the range of IDs to allocate.
-    rpc: datastore.RPC to use for this request.
+    config: Optional Configuration to use for this request.
 
   Returns:
     (start, end) of the allocated range, inclusive.
   """
   max = kwargs.pop('max', None)
-  rpc = GetRpcFromKwargs(kwargs)
+  config = _GetConfigFromKwargs(kwargs)
+  if getattr(config, 'read_policy', None) == EVENTUAL_CONSISTENCY:
+    raise datastore_errors.BadRequestError(
+        'read_policy is only supported on read operations.')
   keys, _ = NormalizeAndTypeCheckKeys(model_key)
 
   if len(keys) > 1:
     raise datastore_errors.BadArgumentError(
         'Cannot allocate IDs for more than one model key at a time')
 
-  req = datastore_pb.AllocateIdsRequest()
-  if size is not None:
-    if max is not None:
-      raise datastore_errors.BadArgumentError(
-          'Cannot allocate ids using both size and max')
-    if size > _MAX_ID_BATCH_SIZE:
-      raise datastore_errors.BadArgumentError(
-          'Cannot allocate more than %s ids at a time; received %s'
-          % (_MAX_ID_BATCH_SIZE, size))
-    if size <= 0:
-      raise datastore_errors.BadArgumentError(
-          'Cannot allocate less than 1 id; received %s' % size)
-    req.set_size(size)
-  if max:
-    if max < 0:
-      raise datastore_errors.BadArgumentError(
-          'Cannot allocate a range with a max less than 0 id; received %s' %
-          size)
-    req.set_max(max)
-
-  req.mutable_model_key().CopyFrom(keys[0]._ToPb())
-
-  try:
-    resp = _MakeSyncCall('datastore_v3', 'AllocateIds', req,
-                         datastore_pb.AllocateIdsResponse(), rpc)
-  except apiproxy_errors.ApplicationError, err:
-    raise _ToDatastoreError(err)
-
-  return resp.start(), resp.end()
+  rpc = _GetConnection().async_allocate_ids(config, keys[0], size, max)
+  return rpc.get_result()
 
 
 class MultiQuery(Query):
@@ -1642,17 +1664,17 @@
       limit: maximum number of values to return.
       offset: offset requested -- if nonzero, this will override the offset in
               the original query
-      rpc: datastore.RPC to use for this request.
+      config: Optional Configuration to use for this request.
 
     Returns:
       A list of entities with at most "limit" entries (less if the query
       completes before reading limit values).
     """
-    rpc = GetRpcFromKwargs(kwargs)
+    config = _GetConfigFromKwargs(kwargs)
     count = 1
     result = []
 
-    iterator = self.Run(rpc=rpc)
+    iterator = self.Run(config=config)
 
     try:
       for i in xrange(offset):
@@ -1782,17 +1804,14 @@
     Merge sort the results. First create a list of iterators, then walk
     though them and yield results in order.
     """
-    rpc = GetRpcFromKwargs(kwargs)
+    config = _GetConfigFromKwargs(kwargs)
+    config = _Rpc2Config(config)
     results = []
     count = 1
     log_level = logging.DEBUG - 1
     for bound_query in self.__bound_queries:
       logging.log(log_level, 'Running query #%i' % count)
-      if rpc:
-        rpc_clone = rpc.clone()
-      else:
-        rpc_clone = None
-      results.append(bound_query.Run(rpc=rpc_clone))
+      results.append(bound_query.Run(config=config))
       count += 1
 
     def IterateResults(results):
@@ -1852,25 +1871,27 @@
     Args:
       limit: maximum number of entries to count (for any result > limit, return
       limit).
-      rpc: datastore.RPC to use for this request.
+      config: Optional Configuration to use for this request.
 
     Returns:
       count of the number of entries returned.
     """
-    rpc = GetRpcFromKwargs(kwargs)
+    config = _GetConfigFromKwargs(kwargs)
     if limit is None:
       count = 0
-      for i in self.Run(rpc=rpc):
+      for _ in self.Run(config=config):
         count += 1
       return count
     else:
-      return len(self.Get(limit, rpc=rpc))
+      return len(self.Get(limit, config=config))
 
-  def GetCompiledCursor(self):
+  def GetCursor(self):
     raise AssertionError('No cursor available for a MultiQuery (queries '
                          'using "IN" or "!=" operators)')
 
-  def GetCompiledQuery(self):
+
+  def _GetCompiledQuery(self):
+    """Internal only, do not use."""
     raise AssertionError('No compilation available for a MultiQuery (queries '
                          'using "IN" or "!=" operators)')
 
@@ -1934,253 +1955,8 @@
   def __iter__(self):
     return iter(self.__bound_queries)
 
-
-
-class Iterator(object):
-  """An iterator over the results of a datastore query.
-
-  Iterators are used to access the results of a Query. An iterator is
-  obtained by building a Query, then calling Run() on it.
-
-  Iterator implements Python's iterator protocol, so results can be accessed
-  with the for and in statements:
-
-  > it = Query('Person').Run()
-  > for person in it:
-  >   print 'Hi, %s!' % person['name']
-  """
-  def __init__(self, query_result_pb, batch_size=None, rpc=None,
-               query_request_pb=None):
-    """Constructor.
-
-    kwargs gets stored and passed on to Next calls made by this iterator.
-    """
-    self.__cursor = query_result_pb.cursor()
-    self.__keys_only = query_result_pb.keys_only()
-    self.__batch_size = batch_size
-    self.__rpc = rpc
-    self.__skipped_results = 0
-
-    self.__results_since_prev = 0
-    self.__prev_compiled_cursor = None
-    self.__next_compiled_cursor = None
-
-    if query_request_pb:
-      self.__remaining_offset = query_request_pb.offset()
-    else:
-      self.__remaining_offset = 0
-
-    if query_request_pb and query_result_pb.has_compiled_cursor():
-      if query_request_pb.has_compiled_cursor():
-        self.__next_compiled_cursor = query_request_pb.compiled_cursor()
-      else:
-        self.__next_compiled_cursor = datastore_pb.CompiledCursor()
-      self.__buffer = self._ProcessQueryResult(query_result_pb)
-      self.__results_since_prev = query_request_pb.offset()
-    else:
-      self.__buffer = self._ProcessQueryResult(query_result_pb)
-
-  def _Get(self, count):
-    """Gets the next count result(s) of the query.
-
-    Not intended to be used by application developers. Use the python
-    iterator protocol instead.
-
-    This method uses _Next to returns the next entities or keys from the list of
-    matching results. If the query specified a sort order, results are returned
-    in that order. Otherwise, the order is undefined.
-
-    The argument, count, specifies the number of results to return. However, the
-    length of the returned list may be smaller than count. This is the case only
-    if count is greater than the number of remaining results.
-
-    The results are always returned as a list. If there are no results left,
-    an empty list is returned.
-
-    Args:
-      # the number of results to return; must be >= 1
-      count: int or long
-
-    Returns:
-      # a list of entities or keys
-      [Entity or Key, ...]
-    """
-    entity_list = self._Next(count)
-    while len(entity_list) < count and self.__more_results:
-      entity_list += self._Next(count - len(entity_list))
-    return entity_list;
-
-  def _Next(self, count=None):
-    """Returns the next batch of results.
-
-    Not intended to be used by application developers. Use the python
-    iterator protocol instead.
-
-    Values are returned in the order they are recieved from the datastore.
-
-    If there are values in the internal buffer they are returned, otherwise a
-    single RPC is run in an attempt to fulfill the request.
-
-    The optional argument, count, specifies the number of results to return.
-    However, the length of the returned list may be smaller than count. This is
-    the case if:
-     - the local buffer has results and count is greater than the number of
-       results in the buffer.
-     - count is greater than the number of remaining results
-     - the size of the remaining results exceeds the RPC buffer limit
-    Use _Get to ensure all possible entities are retrieved.
-
-    When count is None, if there are items in the local buffer, they are
-    all returned, otherwise the datastore backend is allowed to decide how many
-    entities to send.
-
-    The internal buffer is also used by the next() method so it is best not to
-    mix _Next() and next().
-
-    The results are always returned as a list. If there are results left, at
-    least one result will be returned in this list. If there are no results
-    left, an empty list is returned.
-
-    Args:
-      # the number of results to return; must be >= 1
-      count: int or long or None
-
-    Returns:
-      # a list of entities or keys
-      [Entity or Key, ...]
-    """
-    if count is not None and (not isinstance(count, (int, long)) or count < 0):
-      raise datastore_errors.BadArgumentError(
-        'Argument to _Next must be an int greater than or equal to 0; received '
-        '%s (a %s)' % (count, typename(count)))
-
-    if self.__buffer:
-      if count is None:
-        entity_list = self.__buffer
-        self.__buffer = []
-      elif count <= len(self.__buffer):
-        entity_list = self.__buffer[:count]
-        del self.__buffer[:count]
-      else:
-        entity_list = self.__buffer
-        self.__buffer = []
-      self.__results_since_prev += len(entity_list)
-      return entity_list
-
-
-    if not self.__more_results:
-      return []
-
-    req = datastore_pb.NextRequest()
-    if self.__remaining_offset:
-      req.set_offset(self.__remaining_offset)
-    if count is not None:
-      req.set_count(count)
-    if self.__next_compiled_cursor:
-      req.set_compile(True)
-    req.mutable_cursor().CopyFrom(self.__cursor)
-    try:
-      rpc = self.__rpc
-      if rpc:
-        self.__rpc = rpc.clone()
-
-      result = _MakeSyncCall('datastore_v3', 'Next', req,
-                             datastore_pb.QueryResult(), rpc)
-    except apiproxy_errors.ApplicationError, err:
-      raise _ToDatastoreError(err)
-
-    new_batch = self._ProcessQueryResult(result)
-    if not self.__has_advanced:
-      self.__more_results = False
-    return new_batch
-
-  def _ProcessQueryResult(self, result):
-    """Returns all results from datastore_pb.QueryResult and updates
-    self.__more_results
-
-    Not intended to be used by application developers. Use the python
-    iterator protocol instead.
-
-    The results are always returned as a list. If there are no results left,
-    an empty list is returned.
-
-    Args:
-      # the instance of datastore_pb.QueryResult to be stored
-      result: datastore_pb.QueryResult
-
-    Returns:
-      # a list of entities or keys
-      [Entity or Key, ...]
-    """
-    if self.__next_compiled_cursor and result.has_compiled_cursor():
-      self.__prev_compiled_cursor = self.__next_compiled_cursor
-      self.__next_compiled_cursor = result.compiled_cursor()
-      self.__results_since_prev = 0
-
-    self.__more_results = result.more_results()
-    if result.skipped_results():
-      self.__has_advanced = True
-      self.__skipped_results += result.skipped_results()
-      self.__remaining_offset -= result.skipped_results()
-    else:
-      self.__has_advanced = result.result_size() > 0
-
-    if self.__keys_only:
-      return [Key._FromPb(e.key()) for e in result.result_list()]
-    else:
-      return [Entity._FromPb(e) for e in result.result_list()]
-
-  def _SkippedResults(self):
-    self.__PrepBuffer()
-    return self.__skipped_results
-
-  def GetCompiledCursor(self, query):
-    if not self.__buffer:
-      return self.__next_compiled_cursor
-    elif not self.__results_since_prev:
-      return self.__prev_compiled_cursor
-    elif self.__prev_compiled_cursor:
-      return Query._RunInternal(query._ToPb(limit=0,
-                                            offset=self.__results_since_prev),
-                                rpc=self.__rpc)[0].GetCompiledCursor(query)
-    else:
-      return None
-
-  def next(self):
-    self.__PrepBuffer()
-    try:
-      result = self.__buffer.pop(0)
-    except IndexError:
-      raise StopIteration
-    self.__results_since_prev += 1
-    return result
-
-  def __PrepBuffer(self):
-    """Loads the next set of values into the local buffer if needed."""
-    while not self.__buffer and self.__more_results:
-      self.__buffer = self._Next(self.__batch_size)
-
-  def __iter__(self): return self
-
-class _Transaction(object):
-  """Encapsulates a transaction currently in progress.
-
-  If we've sent a BeginTransaction call, then handle will be a
-  datastore_pb.Transaction that holds the transaction handle.
-
-  If we know the entity group for this transaction, it's stored in the
-  entity_group attribute, which is set by RunInTransaction().
-
-  modified_keys is a set containing the Keys of all entities modified (ie put
-  or deleted) in this transaction. If an entity is modified more than once, a
-  BadRequestError is raised.
-  """
-  def __init__(self):
-    """Initializes modified_keys to the empty set."""
-    self.handle = None
-    self.entity_group = None
-    self.modified_keys = None
-    self.modified_keys = set()
+  GetCompiledCursor = GetCursor
+  GetCompiledQuery = _GetCompiledQuery
 
 
 def RunInTransaction(function, *args, **kwargs):
@@ -2211,7 +1987,8 @@
   Runs the user-provided function inside a full-featured, ACID datastore
   transaction. Every Put, Get, and Delete call in the function is made within
   the transaction. All entities involved in these calls must belong to the
-  same entity group. Queries are not supported.
+  same entity group. Queries are supported as long as they specify an
+  ancestor belonging to the same entity group.
 
   The trailing arguments are passed to the function as positional arguments.
   If the function returns a value, that value will be returned by
@@ -2260,7 +2037,7 @@
   Nested transactions are not supported.
 
   Args:
-    # number of retries
+    # number of retries (not counting the initial try)
     retries: integer
     # a function to be run inside the transaction
     function: callable
@@ -2274,142 +2051,78 @@
     TransactionFailedError, if the transaction could not be committed.
   """
 
-  if _CurrentTransactionKey():
-    raise datastore_errors.BadRequestError(
-      'Nested transactions are not supported.')
-
   if retries < 0:
     raise datastore_errors.BadRequestError(
       'Number of retries should be non-negative number.')
 
-  tx_key = None
+  if IsInTransaction():
+    raise datastore_errors.BadRequestError(
+      'Nested transactions are not supported.')
+
+  old_connection = _GetConnection()
+  for i in range(0, retries + 1):
+    new_connection = old_connection.new_transaction()
+    _SetConnection(new_connection)
+    try:
+      ok, result = _DoOneTry(new_connection, function, args, kwargs)
+      if ok:
+        return result
+    finally:
+      _SetConnection(old_connection)
+
+  raise datastore_errors.TransactionFailedError(
+    'The transaction could not be committed. Please try again.')
+
+
+def _DoOneTry(new_connection, function, args, kwargs):
+  """Helper to call a function in a transaction, once.
+
+  Args:
+    new_connection: The new, transactional, connection object.
+    function: The function to call.
+    args: Tuple of positional arguments.
+    kwargs: Dict of keyword arguments.
+  """
 
   try:
-    tx_key = _NewTransactionKey()
-    tx = _Transaction()
-    _txes[tx_key] = tx
-
-    for i in range(0, retries + 1):
-      tx.modified_keys.clear()
-
-      try:
-        result = function(*args, **kwargs)
-      except:
-        original_exception = sys.exc_info()
-
-        if tx.handle:
-          try:
-            _MakeSyncCall('datastore_v3', 'Rollback',
-                          tx.handle, api_base_pb.VoidProto())
-          except:
-            logging.info('Exception sending Rollback:\n' +
-                         traceback.format_exc())
-
-        type, value, trace = original_exception
-        if type is datastore_errors.Rollback:
-          return
-        else:
-          raise type, value, trace
-
-      if tx.handle:
-        try:
-          _MakeSyncCall('datastore_v3', 'Commit',
-                        tx.handle, datastore_pb.CommitResponse())
-        except apiproxy_errors.ApplicationError, err:
-          if (err.application_error ==
-              datastore_pb.Error.CONCURRENT_TRANSACTION):
-            logging.warning('Transaction collision for entity group with '
-                            'key %r. Retrying...', tx.entity_group)
-            tx.handle = None
-            tx.entity_group = None
-            continue
-          else:
-            raise _ToDatastoreError(err)
-
-      return result
-
-    raise datastore_errors.TransactionFailedError(
-      'The transaction could not be committed. Please try again.')
-
-  finally:
-    if tx_key in _txes:
-      del _txes[tx_key]
-    del tx_key
+    result = function(*args, **kwargs)
+
+  except:
+    original_exception = sys.exc_info()
+
+    try:
+      new_connection.rollback()
+    except Exception:
+      logging.exception('Exception sending Rollback:')
+
+    type, value, trace = original_exception
+    if isinstance(value, datastore_errors.Rollback):
+      return True, None
+    else:
+      raise type, value, trace
+
+  else:
+    if new_connection.commit():
+      return True, result
+    else:
+      logging.warning('Transaction collision. Retrying... %s', '')
+      return False, None
 
 
 def _MaybeSetupTransaction(request, keys):
-  """Begins a transaction, if necessary, and populates it in the request.
-
-  If we're currently inside a transaction, this records the entity group,
-  checks that the keys are all in that entity group, creates the transaction
-  PB, and sends the BeginTransaction. It then populates the transaction handle
-  in the request.
-
-  Raises BadRequestError if the entity has a different entity group than the
-  current transaction.
+  """Begin a transaction, if necessary, and populate it in the request.
+
+  This API exists for internal backwards compatibility, primarily with
+  api/taskqueue/taskqueue.py.
 
   Args:
-    request: GetRequest, PutRequest, DeleteRequest, or Query
-    keys: sequence of Keys
+    request: A protobuf with a mutable_transaction() method.
+    keys: Unused.
 
   Returns:
-    _Transaction if we're inside a transaction, otherwise None
+    A transaction if we're inside a transaction, otherwise None
   """
-  assert isinstance(request, (datastore_pb.GetRequest, datastore_pb.PutRequest,
-                              datastore_pb.DeleteRequest, datastore_pb.Query,
-                              taskqueue_service_pb.TaskQueueAddRequest,
-                              )), request.__class__
-  tx_key = None
-
-  try:
-    tx_key = _CurrentTransactionKey()
-    if tx_key:
-      tx = _txes[tx_key]
-
-      groups = [k.entity_group() for k in keys]
-      if tx.entity_group:
-        expected_group = tx.entity_group
-      elif groups:
-        expected_group = groups[0]
-      else:
-        expected_group = None
-
-      for group in groups:
-        if (group != expected_group or
-
-
-
-
-
-
-
-            (not group.has_id_or_name() and group is not expected_group)):
-          raise _DifferentEntityGroupError(expected_group, group)
-
-        if not tx.entity_group and group.has_id_or_name():
-          tx.entity_group = group
-
-      if not tx.handle:
-        req = datastore_pb.BeginTransactionRequest()
-        if keys:
-          req.set_app(keys[0].app())
-        else:
-          assert isinstance(request, taskqueue_service_pb.TaskQueueAddRequest)
-          req.set_app(os.environ['APPLICATION_ID'])
-        assert req.app()
-
-        tx.handle = _MakeSyncCall('datastore_v3', 'BeginTransaction',
-                                  req, datastore_pb.Transaction())
-
-        if not tx.handle.app():
-          tx.handle.set_app(req.app())
-
-      request.mutable_transaction().CopyFrom(tx.handle)
-
-      return tx
-
-  finally:
-    del tx_key
+  return _GetConnection()._set_request_transaction(request)
 
 
 def IsInTransaction():
@@ -2418,52 +2131,7 @@
   Returns:
     True if already running in transaction, else False.
   """
-  return bool(_CurrentTransactionKey())
-
-
-def _DifferentEntityGroupError(a, b):
-  """Raises a BadRequestError that says the given entity groups are different.
-
-  Includes the two entity groups in the message, formatted more clearly and
-  concisely than repr(Key).
-
-  Args:
-    a, b are both Keys that represent entity groups.
-  """
-  def id_or_name(key):
-    if key.name():
-      return 'name=%r' % key.name()
-    else:
-      return 'id=%r' % key.id()
-
-  raise datastore_errors.BadRequestError(
-    'Cannot operate on different entity groups in a transaction: '
-    '(kind=%r, %s) and (kind=%r, %s).' % (a.kind(), id_or_name(a),
-                                          b.kind(), id_or_name(b)))
-
-
-def _FindTransactionFrameInStack():
-  """Walks the stack to find a RunInTransaction() call.
-
-  Returns:
-    # this is the RunInTransactionCustomRetries() frame record, if found
-    frame record or None
-  """
-  frame = sys._getframe()
-  filename = frame.f_code.co_filename
-
-  frame = frame.f_back.f_back
-  while frame:
-    if (frame.f_code.co_filename == filename and
-        frame.f_code.co_name == 'RunInTransactionCustomRetries'):
-      return frame
-    frame = frame.f_back
-
-  return None
-
-_CurrentTransactionKey = _FindTransactionFrameInStack
-
-_NewTransactionKey = sys._getframe
+  return isinstance(_GetConnection(), datastore_rpc.TransactionalConnection)
 
 
 def _GetCompleteKeyOrError(arg):
@@ -2541,44 +2209,29 @@
     dictionary[key] = value
 
 
-def _ToDatastoreError(err):
-  """Converts an apiproxy.ApplicationError to an error in datastore_errors.
-
-  Args:
-    err: apiproxy.ApplicationError
-
-  Returns:
-    a subclass of datastore_errors.Error
-  """
-  return _DatastoreExceptionFromErrorCodeAndDetail(err.application_error,
-                                                   err.error_detail)
-
-
-def _DatastoreExceptionFromErrorCodeAndDetail(error, detail):
-  """Converts a datastore_pb.Error into a datastore_errors.Error.
-
-  Args:
-    error: A member of the datastore_pb.Error enumeration.
-    detail: A string providing extra details about the error.
-
-  Returns:
-    A subclass of datastore_errors.Error.
-  """
-  exception_class = {
-      datastore_pb.Error.BAD_REQUEST: datastore_errors.BadRequestError,
-      datastore_pb.Error.CONCURRENT_TRANSACTION:
-          datastore_errors.TransactionFailedError,
-      datastore_pb.Error.INTERNAL_ERROR: datastore_errors.InternalError,
-      datastore_pb.Error.NEED_INDEX: datastore_errors.NeedIndexError,
-      datastore_pb.Error.TIMEOUT: datastore_errors.Timeout,
-      datastore_pb.Error.BIGTABLE_ERROR: datastore_errors.Timeout,
-      datastore_pb.Error.COMMITTED_BUT_STILL_APPLYING:
-          datastore_errors.CommittedButStillApplying,
-      datastore_pb.Error.CAPABILITY_DISABLED:
-          apiproxy_errors.CapabilityDisabledError,
-  }.get(error, datastore_errors.Error)
-
-  if detail is None:
-    return exception_class()
-  else:
-    return exception_class(detail)
+class Iterator(datastore_query.ResultsIterator):
+  """Thin wrapper of datastore_query.ResultsIterator.
+
+  Deprecated, do not use, only for backwards compatability.
+  """
+  def _Next(self, count=None):
+    if count is None:
+      count = 20
+    result = []
+    for r in self:
+      if len(result) >= count:
+        break;
+      result.append(r)
+    return result
+
+  def GetCompiledCursor(self, query):
+    return self.cursor()
+
+  _Get = _Next
+
+
+DatastoreRPC = apiproxy_stub_map.UserRPC
+GetRpcFromKwargs = _GetConfigFromKwargs
+_CurrentTransactionKey = IsInTransaction
+_ToDatastoreError = datastore_rpc._ToDatastoreError
+_DatastoreExceptionFromErrorCodeAndDetail = datastore_rpc._DatastoreExceptionFromErrorCodeAndDetail
 
=== modified file 'AppServer/google/appengine/api/datastore_distributed.py'
--- AppServer/google/appengine/api/datastore_distributed.py	2010-12-17 22:47:53 +0000
+++ AppServer/google/appengine/api/datastore_distributed.py	2010-12-24 09:11:16 +0000
@@ -58,9 +58,9 @@
 
 SSL_DEFAULT_PORT = 8443
 try:
-  __import__('google.appengine.api.labs.taskqueue.taskqueue_service_pb')
+  __import__('google.appengine.api.taskqueue.taskqueue_service_pb')
   taskqueue_service_pb = sys.modules.get(
-      'google.appengine.api.labs.taskqueue.taskqueue_service_pb')
+      'google.appengine.api.taskqueue.taskqueue_service_pb')
 except ImportError:
   from google.appengine.api.taskqueue import taskqueue_service_pb
 
@@ -528,18 +528,10 @@
   def _RemoteSend(self, request, response, method):
     tag = self.__app_id
     user = users.GetCurrentUser()
-    APPSCALE_VERSION = '1'
-    try:
-      APPSCALE_VERSION = os.environ['APPSCALE_VERSION']
-    except Exception, e:
-      logging.info("WARNING: Appscale version secret not set")
-
     if user != None:
       tag += ":" + user.email()
       tag += ":" + user.nickname()
       tag += ":" + user.auth_domain()
-    if APPSCALE_VERSION:
-      tag += ":" + APPSCALE_VERSION
     api_request = remote_api_pb.Request()
     api_request.set_method(method)
     api_request.set_service_name("datastore_v3")
 
=== modified file 'AppServer/google/appengine/api/datastore_file_stub.py'
--- AppServer/google/appengine/api/datastore_file_stub.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/datastore_file_stub.py	2010-12-24 09:11:16 +0000
@@ -42,7 +42,6 @@
 import sys
 import tempfile
 import threading
-import warnings
 
 import cPickle as pickle
 
@@ -53,6 +52,7 @@
 from google.appengine.api import datastore_errors
 from google.appengine.api import datastore_types
 from google.appengine.api import users
+from google.appengine.api.taskqueue import taskqueue_service_pb
 from google.appengine.datastore import datastore_pb
 from google.appengine.datastore import datastore_index
 from google.appengine.datastore import datastore_stub_util
@@ -60,24 +60,12 @@
 from google.net.proto import ProtocolBuffer
 from google.appengine.datastore import entity_pb
 
-try:
-  __import__('google.appengine.api.labs.taskqueue.taskqueue_service_pb')
-  taskqueue_service_pb = sys.modules.get(
-      'google.appengine.api.labs.taskqueue.taskqueue_service_pb')
-except ImportError:
-  from google.appengine.api.taskqueue import taskqueue_service_pb
 
 entity_pb.Reference.__hash__ = lambda self: hash(self.Encode())
 datastore_pb.Query.__hash__ = lambda self: hash(self.Encode())
 datastore_pb.Transaction.__hash__ = lambda self: hash(self.Encode())
 
 
-_MAXIMUM_RESULTS = 1000
-
-
-_MAX_QUERY_OFFSET = 1000
-
-
 _MAX_QUERY_COMPONENTS = 100
 
 
@@ -87,9 +75,6 @@
 _MAX_ACTIONS_PER_TXN = 5
 
 
-_CURSOR_CONCAT_STR = '!CURSOR!'
-
-
 class _StoredEntity(object):
   """Simple wrapper around an entity stored by the stub.
 
@@ -109,266 +94,8 @@
 
     self.encoded_protobuf = entity.Encode()
 
-    self.native = datastore.Entity._FromPb(entity)
-
-
-class _Cursor(object):
-  """A query cursor.
-
-  Public properties:
-    cursor: the integer cursor
-    count: the original total number of results
-    keys_only: whether the query is keys_only
-    app: the app for which this cursor was created
-
-  Class attributes:
-    _next_cursor: the next cursor to allocate
-    _next_cursor_lock: protects _next_cursor
-  """
-  _next_cursor = 1
-  _next_cursor_lock = threading.Lock()
-
-  def __init__(self, query, results, order_compare_entities):
-    """Constructor.
-
-    Args:
-      query: the query request proto
-      # the query results, in order, such that results[self.offset+1] is
-      # the next result
-      results: list of datastore.Entity
-      order_compare_entities: a __cmp__ function for datastore.Entity that
-        follows sort order as specified by the query
-    """
-
-    if query.has_compiled_cursor() and query.compiled_cursor().position_list():
-      (self.__last_result, inclusive) = self._DecodeCompiledCursor(
-          query, query.compiled_cursor())
-      start_cursor_position = _Cursor._GetCursorOffset(results,
-                                                       self.__last_result,
-                                                       inclusive,
-                                                       order_compare_entities)
-    else:
-      self.__last_result = None
-      start_cursor_position = 0
-
-    if query.has_end_compiled_cursor():
-      (end_cursor_entity, inclusive) = self._DecodeCompiledCursor(
-          query, query.end_compiled_cursor())
-      end_cursor_position = _Cursor._GetCursorOffset(results,
-                                                     end_cursor_entity,
-                                                     inclusive,
-                                                     order_compare_entities)
-    else:
-      end_cursor_position = len(results)
-
-    results = results[start_cursor_position:end_cursor_position]
-
-    if query.has_limit():
-      limit = query.limit()
-      if query.offset():
-        limit += query.offset()
-      if limit > 0 and limit < len(results):
-        results = results[:limit]
-
-    self.__results = results
-    self.__query = query
-    self.__offset = 0
-
-    self.app = query.app()
-    self.keys_only = query.keys_only()
-    self.count = len(self.__results)
-    self.cursor = self._AcquireCursorID()
-
-  def _AcquireCursorID(self):
-    """Acquires the next cursor id in a thread safe manner.
-    """
-    self._next_cursor_lock.acquire()
-    try:
-      cursor_id = _Cursor._next_cursor
-      _Cursor._next_cursor += 1
-    finally:
-      self._next_cursor_lock.release()
-    return cursor_id
-
-  @staticmethod
-  def _GetCursorOffset(results, cursor_entity, inclusive, compare):
-    """Converts a cursor entity into a offset into the result set even if the
-    cursor_entity no longer exists.
-
-    Args:
-      cursor_entity: the decoded datastore.Entity from the compiled query
-      inclusive: boolean that specifies if to offset past the cursor_entity
-      compare: a function that takes two datastore.Entity and compares them
-    Returns:
-      the integer offset
-    """
-    lo = 0
-    hi = len(results)
-    if inclusive:
-      while lo < hi:
-        mid = (lo + hi) // 2
-        if compare(results[mid], cursor_entity) < 0:
-          lo = mid + 1
-        else:
-          hi = mid
-    else:
-      while lo < hi:
-        mid = (lo + hi) // 2
-        if compare(cursor_entity, results[mid]) < 0:
-          hi = mid
-        else:
-          lo = mid + 1
-    return lo
-
-  def _ValidateQuery(self, query, query_info):
-    """Ensure that the given query matches the query_info.
-
-    Args:
-      query: datastore_pb.Query instance we are chacking
-      query_info: datastore_pb.Query instance we want to match
-
-    Raises BadRequestError on failure.
-    """
-    error_msg = 'Cursor does not match query: %s'
-    exc = datastore_errors.BadRequestError
-    if query_info.filter_list() != query.filter_list():
-      raise exc(error_msg % 'filters do not match')
-    if query_info.order_list() != query.order_list():
-      raise exc(error_msg % 'orders do not match')
-
-    for attr in ('ancestor', 'kind', 'name_space', 'search_query'):
-      query_info_has_attr = getattr(query_info, 'has_%s' % attr)
-      query_info_attr = getattr(query_info, attr)
-      query_has_attr = getattr(query, 'has_%s' % attr)
-      query_attr = getattr(query, attr)
-      if query_info_has_attr():
-        if not query_has_attr() or query_info_attr() != query_attr():
-          raise exc(error_msg % ('%s does not match' % attr))
-      elif query_has_attr():
-        raise exc(error_msg % ('%s does not match' % attr))
-
-  def _MinimalQueryInfo(self, query):
-    """Extract the minimal set of information for query matching.
-
-    Args:
-      query: datastore_pb.Query instance from which to extract info.
-
-    Returns:
-      datastore_pb.Query instance suitable for matching against when
-      validating cursors.
-    """
-    query_info = datastore_pb.Query()
-    query_info.set_app(query.app())
-
-    for filter in query.filter_list():
-      query_info.filter_list().append(filter)
-    for order in query.order_list():
-      query_info.order_list().append(order)
-
-    if query.has_ancestor():
-      query_info.mutable_ancestor().CopyFrom(query.ancestor())
-
-    for attr in ('kind', 'name_space', 'search_query'):
-      query_has_attr = getattr(query, 'has_%s' % attr)
-      query_attr = getattr(query, attr)
-      query_info_set_attr = getattr(query_info, 'set_%s' % attr)
-      if query_has_attr():
-        query_info_set_attr(query_attr())
-
-    return query_info
-
-  def _MinimalEntityInfo(self, entity_proto, query):
-    """Extract the minimal set of information that preserves entity order.
-
-    Args:
-      entity_proto: datastore_pb.EntityProto instance from which to extract
-      information
-      query: datastore_pb.Query instance for which ordering must be preserved.
-
-    Returns:
-      datastore_pb.EntityProto instance suitable for matching against a list of
-      results when finding cursor positions.
-    """
-    entity_info = datastore_pb.EntityProto();
-    order_names = [o.property() for o in query.order_list()]
-    entity_info.mutable_key().MergeFrom(entity_proto.key())
-    entity_info.mutable_entity_group().MergeFrom(entity_proto.entity_group())
-    for prop in entity_proto.property_list():
-      if prop.name() in order_names:
-        entity_info.add_property().MergeFrom(prop)
-    return entity_info;
-
-  def _DecodeCompiledCursor(self, query, compiled_cursor):
-    """Converts a compiled_cursor into a cursor_entity.
-
-    Returns:
-      (cursor_entity, inclusive): a datastore.Entity and if it should be
-      included in the result set.
-    """
-    assert len(compiled_cursor.position_list()) == 1
-
-    position = compiled_cursor.position(0)
-    entity_pb = datastore_pb.EntityProto()
-    (query_info_encoded, entity_encoded) = position.start_key().split(
-        _CURSOR_CONCAT_STR, 1)
-    query_info_pb = datastore_pb.Query()
-    query_info_pb.ParseFromString(query_info_encoded)
-    self._ValidateQuery(query, query_info_pb)
-
-    entity_pb.ParseFromString(entity_encoded)
-    return (datastore.Entity._FromPb(entity_pb, True),
-            position.start_inclusive())
-
-  def _EncodeCompiledCursor(self, query, compiled_cursor):
-    """Converts the current state of the cursor into a compiled_cursor
-
-    Args:
-      query: the datastore_pb.Query this cursor is related to
-      compiled_cursor: an empty datstore_pb.CompiledCursor
-    """
-    if self.__last_result is not None:
-      position = compiled_cursor.add_position()
-      query_info = self._MinimalQueryInfo(query)
-      entity_info = self._MinimalEntityInfo(self.__last_result.ToPb(), query)
-      start_key = _CURSOR_CONCAT_STR.join((
-          query_info.Encode(),
-          entity_info.Encode()))
-      position.set_start_key(str(start_key))
-      position.set_start_inclusive(False)
-
-  def PopulateQueryResult(self, result, count, offset, compile=False):
-    """Populates a QueryResult with this cursor and the given number of results.
-
-    Args:
-      result: datastore_pb.QueryResult
-      count: integer of how many results to return
-      offset: integer of how many results to skip
-      compile: boolean, whether we are compiling this query
-    """
-    offset = min(offset, self.count - self.__offset)
-    limited_offset = min(offset, _MAX_QUERY_OFFSET)
-    if limited_offset:
-      self.__offset += limited_offset
-      result.set_skipped_results(limited_offset)
-
-    if offset == limited_offset and count:
-      if count > _MAXIMUM_RESULTS:
-        count = _MAXIMUM_RESULTS
-      results = self.__results[self.__offset:self.__offset + count]
-      count = len(results)
-      self.__offset += count
-      result.result_list().extend(r._ToPb() for r in results)
-
-    if self.__offset:
-      self.__last_result = self.__results[self.__offset - 1]
-
-    result.mutable_cursor().set_app(self.app)
-    result.mutable_cursor().set_cursor(self.cursor)
-    result.set_keys_only(self.keys_only)
-    result.set_more_results(self.__offset < self.count)
-    if compile:
-      self._EncodeCompiledCursor(
-          self.__query, result.mutable_compiled_cursor())
+    self.native = datastore.Entity._FromPb(entity,
+                                           validate_reserved_properties=False)
 
 
 class KindPseudoKind(object):
@@ -381,10 +108,48 @@
   """
   name = '__kind__'
 
+  def Query(self, entities, query, filters, orders):
+    """Perform a query on this pseudo-kind.
+
+    Args:
+      entities: all the app's entities.
+      query: the original datastore_pb.Query.
+      filters: the filters from query.
+      orders: the orders from query.
+
+    Returns:
+      (results, remaining_filters, remaining_orders)
+      results is a list of datastore.Entity
+      remaining_filters and remaining_orders are the filters and orders that
+      should be applied in memory
+    """
+    kind_range = datastore_stub_util.ParseKindQuery(query, filters, orders)
+    app_namespace_str = datastore_types.EncodeAppIdNamespace(
+        query.app(), query.name_space())
+    kinds = []
+
+    for app_namespace, kind in entities:
+      if app_namespace != app_namespace_str: continue
+      if not kind_range.Contains(kind): continue
+      kinds.append(datastore.Entity(self.name, name=kind))
+
+    return (kinds, [], [])
+
+
+class PropertyPseudoKind(object):
+  """Pseudo-kind for schema queries.
+
+  Provides a Query method to perform the actual query.
+
+  Public properties:
+    name: the pseudo-kind name
+  """
+  name = '__property__'
+
   def __init__(self, filestub):
     """Constructor.
 
-    Initializes a __kind__ pseudo-kind definition.
+    Initializes a __property__ pseudo-kind definition.
 
     Args:
       filestub: the DatastoreFileStub instance being served by this
@@ -396,10 +161,10 @@
     """Perform a query on this pseudo-kind.
 
     Args:
-      entities: all the app's entities
-      query: the original datastore_pb.Query
-      filters: the filters from query
-      orders: the orders from query
+      entities: all the app's entities.
+      query: the original datastore_pb.Query.
+      filters: the filters from query.
+      orders: the orders from query.
 
     Returns:
       (results, remaining_filters, remaining_orders)
@@ -407,63 +172,69 @@
       remaining_filters and remaining_orders are the filters and orders that
       should be applied in memory
     """
-    start_kind, start_inclusive, end_kind, end_inclusive = (
-        datastore_stub_util.ParseKindQuery(query, filters, orders))
-    keys_only = query.keys_only()
-    app_str = query.app()
-    namespace_str = query.name_space()
-    keys_only = query.keys_only()
-    app_namespace_str = datastore_types.EncodeAppIdNamespace(app_str,
-                                                             namespace_str)
-    kinds = []
+    property_range = datastore_stub_util.ParsePropertyQuery(query, filters,
+                                                            orders)
+    keys_only = query.keys_only()
+    app_namespace_str = datastore_types.EncodeAppIdNamespace(
+        query.app(), query.name_space())
+
+    properties = []
     if keys_only:
-      usekey = '__kind__keys'
+      usekey = '__property__keys'
     else:
-      usekey = '__kind__'
+      usekey = '__property__'
 
     for app_namespace, kind in entities:
       if app_namespace != app_namespace_str: continue
-      if start_kind is not None:
-        if start_inclusive and kind < start_kind: continue
-        if not start_inclusive and kind <= start_kind: continue
-      if end_kind is not None:
-        if end_inclusive and kind > end_kind: continue
-        if not end_inclusive and kind >= end_kind: continue
+
+      (start_cmp, end_cmp) = property_range.MapExtremes(
+          lambda extreme, inclusive, is_end: cmp(kind, extreme[0]))
+      if not((start_cmp is None or start_cmp >= 0) and
+             (end_cmp is None or end_cmp <= 0)):
+        continue
 
       app_kind = (app_namespace_str, kind)
 
-      kind_e = self.filestub._GetSchemaCache(app_kind, usekey)
-      if not kind_e:
-        kind_e = datastore.Entity(self.name, name=kind)
-
-        if not keys_only:
-          props = {}
-
-          for entity in entities[app_kind].values():
-            for prop in entity.protobuf.property_list():
-              prop_name = prop.name()
-              if prop_name not in props:
-                props[prop_name] = set()
-              cls = entity.native[prop_name].__class__
-              tag = self.filestub._PROPERTY_TYPE_TAGS.get(cls)
-              props[prop_name].add(tag)
-
-          properties = []
-          types = []
-          for name in sorted(props):
-            for tag in sorted(props[name]):
-              properties.append(name)
-              types.append(tag)
-          if properties:
-            kind_e['property'] = properties
-          if types:
-            kind_e['representation'] = types
-
-          self.filestub._SetSchemaCache(app_kind, usekey, kind_e)
-
-      kinds.append(kind_e)
-
-    return (kinds, [], [])
+      kind_properties = self.filestub._GetSchemaCache(app_kind, usekey)
+      if not kind_properties:
+        kind_properties = []
+        kind_key = datastore_types.Key.from_path(KindPseudoKind.name, kind)
+        props = {}
+
+        for entity in entities[app_kind].values():
+          for prop in entity.protobuf.property_list():
+            prop_name = prop.name()
+            if (prop_name in
+                datastore_stub_util.GetInvisibleSpecialPropertyNames()):
+              continue
+            if prop_name not in props:
+              props[prop_name] = set()
+            native_value = entity.native[prop_name]
+            if not isinstance(native_value, list):
+              native_value = [native_value]
+            for value in native_value:
+              tag = self.filestub._PROPERTY_TYPE_TAGS.get(value.__class__)
+              if tag is not None:
+                props[prop_name].add(tag)
+              else:
+                logging.warning('Unexpected value of class %s in datastore', value.__class__)
+
+        for prop in sorted(props):
+          property_e = datastore.Entity(self.name, name=prop, parent=kind_key)
+          kind_properties.append(property_e)
+
+          if not keys_only and props[prop]:
+            property_e['property_representation'] = [
+                datastore_stub_util._PROPERTY_TYPE_NAMES[tag]
+                for tag in sorted(props[prop])]
+
+        self.filestub._SetSchemaCache(app_kind, usekey, kind_properties)
+
+      def InQuery(property_e):
+        return property_range.Contains((kind, property_e.key().name()))
+      properties += filter(InQuery, kind_properties)
+
+    return (properties, [], [])
 
 
 class NamespacePseudoKind(object):
@@ -476,25 +247,14 @@
   """
   name = '__namespace__'
 
-  def __init__(self, filestub):
-    """Constructor.
-
-    Initializes a __namespace__ pseudo-kind definition.
-
-    Args:
-      filestub: the DatastoreFileStub instance being served by this
-          pseudo-kind.
-    """
-    self.filestub = filestub
-
   def Query(self, entities, query, filters, orders):
     """Perform a query on this pseudo-kind.
 
     Args:
-      entities: all the app's entities
-      query: the original datastore_pb.Query
-      filters: the filters from query
-      orders: the orders from query
+      entities: all the app's entities.
+      query: the original datastore_pb.Query.
+      filters: the filters from query.
+      orders: the orders from query.
 
     Returns:
       (results, remaining_filters, remaining_orders)
@@ -502,24 +262,16 @@
       remaining_filters and remaining_orders are the filters and orders that
       should be applied in memory
     """
-    start_namespace, start_inclusive, end_namespace, end_inclusive = (
-        datastore_stub_util.ParseNamespaceQuery(query, filters, orders))
+    namespace_range = datastore_stub_util.ParseNamespaceQuery(query, filters,
+                                                              orders)
     app_str = query.app()
 
     namespaces = set()
 
     for app_namespace, kind in entities:
       (app_id, namespace) = datastore_types.DecodeAppIdNamespace(app_namespace)
-      if app_id != app_str: continue
-
-      if start_namespace is not None:
-        if start_inclusive and namespace < start_namespace: continue
-        if not start_inclusive and namespace <= start_namespace: continue
-      if end_namespace is not None:
-        if end_inclusive and namespace > end_namespace: continue
-        if not end_inclusive and namespace >= end_namespace: continue
-
-      namespaces.add(namespace)
+      if app_id == app_str and namespace_range.Contains(namespace):
+        namespaces.add(namespace)
 
     namespace_entities = []
     for namespace in namespaces:
@@ -558,6 +310,8 @@
     datastore_types.PostalAddress: entity_pb.PropertyValue.kstringValue,
     datastore_types.Rating: entity_pb.PropertyValue.kint64Value,
     str: entity_pb.PropertyValue.kstringValue,
+    datastore_types.ByteString: entity_pb.PropertyValue.kstringValue,
+    datastore_types.BlobKey: entity_pb.PropertyValue.kstringValue,
     datastore_types.Text: entity_pb.PropertyValue.kstringValue,
     type(None): 0,
     unicode: entity_pb.PropertyValue.kstringValue,
@@ -635,8 +389,9 @@
     self.__indexes_lock = threading.Lock()
 
     self.__pseudo_kinds = {}
-    self._RegisterPseudoKind(KindPseudoKind(self))
-    self._RegisterPseudoKind(NamespacePseudoKind(self))
+    self._RegisterPseudoKind(KindPseudoKind())
+    self._RegisterPseudoKind(PropertyPseudoKind(self))
+    self._RegisterPseudoKind(NamespacePseudoKind())
 
     self.Read()
 
@@ -826,14 +581,14 @@
 
     return []
 
-  def __WritePickled(self, obj, filename, openfile=file):
+  def __WritePickled(self, obj, filename):
     """Pickles the object and writes it to the given file.
     """
     if not filename or filename == '/dev/null' or not obj:
       return
 
     descriptor, tmp_filename = tempfile.mkstemp(dir=os.path.dirname(filename))
-    tmpfile = openfile(tmp_filename, 'wb')
+    tmpfile = os.fdopen(descriptor, 'wb')
     pickler = pickle.Pickler(tmpfile, protocol=1)
     pickler.fast = True
     pickler.dump(obj)
@@ -921,6 +676,8 @@
         assert (clone.has_entity_group() and
                 clone.entity_group().element_size() > 0)
 
+      datastore_stub_util.PrepareSpecialPropertiesForStore(clone)
+
     self.__entities_lock.acquire()
 
     try:
@@ -956,6 +713,8 @@
 
       if entity:
         group.mutable_entity().CopyFrom(entity)
+        datastore_stub_util.PrepareSpecialPropertiesForLoad(
+            group.mutable_entity())
 
 
   def _Dynamic_Delete(self, delete_request, delete_response):
@@ -1166,6 +925,14 @@
       if cmped == 0:
         return cmp(a.key(), b.key())
 
+    def order_compare_entities_pb(a, b):
+      """ Return a negative, zero or positive number depending on whether
+      entity a is considered smaller than, equal to, or larger than b,
+      according to the query's orderings. a and b are protobuf-encoded
+      entities."""
+      return order_compare_entities(datastore.Entity.FromPb(a),
+                                    datastore.Entity.FromPb(b))
+
     def order_compare_properties(x, y):
       """Return a negative, zero or positive number depending on whether
       property value x is considered smaller than, equal to, or larger than
@@ -1201,7 +968,11 @@
     else:
       self.__query_history[clone] = 1
 
-    cursor = _Cursor(query, results, order_compare_entities)
+    results = [r._ToPb() for r in results]
+    for result in results:
+      datastore_stub_util.PrepareSpecialPropertiesForLoad(result)
+    cursor = datastore_stub_util.ListCursor(query, results,
+                                            order_compare_entities_pb)
     self.__queries[cursor.cursor] = cursor
 
     if query.has_count():
@@ -1243,7 +1014,8 @@
     query_result = datastore_pb.QueryResult()
     self._Dynamic_RunQuery(query, query_result)
     cursor = query_result.cursor().cursor()
-    integer64proto.set_value(min(self.__queries[cursor].count, _MAXIMUM_RESULTS))
+    integer64proto.set_value(min(self.__queries[cursor].Count(),
+                                 datastore_stub_util._MAXIMUM_RESULTS))
     del self.__queries[cursor]
 
   def _Dynamic_BeginTransaction(self, request, transaction):
@@ -1348,6 +1120,9 @@
 
       for entity in self.__entities[app_kind].values():
         for prop in entity.protobuf.property_list():
+          if (prop.name() in
+              datastore_stub_util.GetInvisibleSpecialPropertyNames()):
+            continue
           if prop.name() not in props:
             props[prop.name()] = entity_pb.PropertyValue()
           props[prop.name()].MergeFrom(prop.value())
 
=== modified file 'AppServer/google/appengine/api/datastore_types.py'
--- AppServer/google/appengine/api/datastore_types.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/datastore_types.py	2010-12-24 09:11:16 +0000
@@ -679,7 +679,7 @@
   TERM = 'user-tag'
 
   def __init__(self, tag):
-    super(Category, self).__init__(self, tag)
+    super(Category, self).__init__()
     ValidateString(tag, 'tag')
 
   def ToXml(self):
@@ -701,7 +701,7 @@
   Raises BadValueError if link is not a fully qualified, well-formed URL.
   """
   def __init__(self, link):
-    super(Link, self).__init__(self, link)
+    super(Link, self).__init__()
     ValidateString(link, 'link', max_len=_MAX_LINK_PROPERTY_LENGTH)
 
     scheme, domain, path, params, query, fragment = urlparse.urlparse(link)
@@ -724,7 +724,7 @@
   Raises BadValueError if email is not a valid email address.
   """
   def __init__(self, email):
-    super(Email, self).__init__(self, email)
+    super(Email, self).__init__()
     ValidateString(email, 'email')
 
   def ToXml(self):
@@ -915,7 +915,7 @@
   Raises BadValueError if phone is not a string or subtype.
   """
   def __init__(self, phone):
-    super(PhoneNumber, self).__init__(self, phone)
+    super(PhoneNumber, self).__init__()
     ValidateString(phone, 'phone')
 
   def ToXml(self):
@@ -933,7 +933,7 @@
   Raises BadValueError if address is not a string or subtype.
   """
   def __init__(self, address):
-    super(PostalAddress, self).__init__(self, address)
+    super(PostalAddress, self).__init__()
     ValidateString(address, 'address')
 
   def ToXml(self):
@@ -955,7 +955,7 @@
   MAX = 100
 
   def __init__(self, rating):
-    super(Rating, self).__init__(self, rating)
+    super(Rating, self).__init__()
     if isinstance(rating, float) or isinstance(rating, complex):
       raise datastore_errors.BadValueError(
         'Expected int or long; received %s (a %s).' %
@@ -1505,7 +1505,7 @@
       same type.
 
   Returns:
-    A list of entity_pb.PropertyValue instances.
+    A list of entity_pb.Property instances.
   """
   encoded_name = name.encode('utf-8')
 
 
=== modified file 'AppServer/google/appengine/api/images/__init__.py'
--- AppServer/google/appengine/api/images/__init__.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/images/__init__.py	2010-12-24 09:11:16 +0000
@@ -897,6 +897,8 @@
   return image.histogram()
 
 
+IMG_SERVING_SIZES_LIMIT = 1600
+
 IMG_SERVING_SIZES = [
     32, 48, 64, 72, 80, 90, 94, 104, 110, 120, 128, 144,
     150, 160, 200, 220, 288, 320, 400, 512, 576, 640, 720,
@@ -927,20 +929,8 @@
 
   "http://lh3.ggpht.com/SomeCharactersGoesHere=s32-c"
 
-  Available sizes for resize are:
-  (e.g. "=sX" where X is one of the following values)
-
-  0, 32, 48, 64, 72, 80, 90, 94, 104, 110, 120, 128, 144,
-  150, 160, 200, 220, 288, 320, 400, 512, 576, 640, 720,
-  800, 912, 1024, 1152, 1280, 1440, 1600
-
-  Available sizes for crop are:
-  (e.g. "=sX-c" where X is one of the following values)
-
-  32, 48, 64, 72, 80, 104, 136, 144, 150, 160
-
-  These values are also available as IMG_SERVING_SIZES and
-  IMG_SERVING_CROP_SIZES integer lists.
+  Available sizes are any interger in the range [0, 1600] and is available as
+  IMG_SERVING_SIZES_LIMIT.
 
   Args:
     size: int, size of resulting images
@@ -960,10 +950,7 @@
   if crop and not size:
     raise BadRequestError("Size should be set for crop operation")
 
-  if size and crop and not size in IMG_SERVING_CROP_SIZES:
-    raise UnsupportedSizeError("Unsupported crop size")
-
-  if size and not crop and not size in IMG_SERVING_SIZES:
+  if size and (size > IMG_SERVING_SIZES_LIMIT or size < 0):
     raise UnsupportedSizeError("Unsupported size")
 
   request = images_service_pb.ImagesGetUrlBaseRequest()
@@ -999,4 +986,3 @@
     url += "-c"
 
   return url
-
 
=== modified file 'AppServer/google/appengine/api/images/images_stub.py'
--- AppServer/google/appengine/api/images/images_stub.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/images/images_stub.py	2010-12-24 09:11:16 +0000
@@ -41,6 +41,9 @@
 from google.appengine.runtime import apiproxy_errors
 
 
+MAX_REQUEST_SIZE = 32 << 20
+
+
 def _ArgbToRgbaTuple(argb):
   """Convert from a single ARGB value to a tuple containing RGBA.
 
@@ -89,7 +92,8 @@
       host_prefix: the URL prefix (protocol://host:port) to preprend to
         image urls on a call to GetUrlBase.
     """
-    super(ImagesServiceStub, self).__init__(service_name)
+    super(ImagesServiceStub, self).__init__(service_name,
+                                            max_request_size=MAX_REQUEST_SIZE)
     self._host_prefix = host_prefix
     Image.init()
 
 
=== modified file 'AppServer/google/appengine/api/labs/taskqueue/__init__.py' (properties changed: -x to +x)
--- AppServer/google/appengine/api/labs/taskqueue/__init__.py	2010-05-07 09:58:53 +0000
+++ AppServer/google/appengine/api/labs/taskqueue/__init__.py	2010-12-24 09:11:16 +0000
@@ -15,6 +15,58 @@
 # limitations under the License.
 #
 
-"""Task Queue API module."""
-
-from taskqueue import *
+"""Shim module so that the old labs import path still works."""
+
+
+
+__all__ = [
+
+    'BadTaskStateError', 'BadTransactionState', 'BadTransactionStateError',
+    'DatastoreError', 'DuplicateTaskNameError', 'Error', 'InternalError',
+    'InvalidQueueError', 'InvalidQueueNameError', 'InvalidTaskError',
+    'InvalidTaskNameError', 'InvalidUrlError', 'PermissionDeniedError',
+    'TaskAlreadyExistsError', 'TaskTooLargeError', 'TombstonedTaskError',
+    'TooManyTasksError', 'TransientError', 'UnknownQueueError',
+
+    'MAX_QUEUE_NAME_LENGTH', 'MAX_TASK_NAME_LENGTH', 'MAX_TASK_SIZE_BYTES',
+    'MAX_URL_LENGTH',
+
+    'Queue', 'Task', 'add']
+
+
+import os
+import sys
+import warnings
+
+from google.appengine.api.taskqueue import *
+
+
+if os.environ.get('DATACENTER', None) is None:
+  warnings.warn('google.appengine.api.labs.taskqueue is deprecated, please use '
+                'google.appengine.api.taskqueue', DeprecationWarning,
+                stacklevel=2)
+
+
+def _map_module(module_name):
+  """Map a module from the new path to the labs path.
+
+  Args:
+    module_name: Name of the module to be mapped.
+
+  Raises:
+    ImportError: If the specified module we are mapping from does not exist.
+
+  Returns:
+    The module object of the module that was mapped.
+  """
+  labs_module_name = '%s.%s' % (__name__, module_name)
+  module_prefix = '.'.join(__name__.split('.')[:2])
+  new_module_name = '%s.api.taskqueue.%s' % (module_prefix, module_name)
+
+  __import__(new_module_name)
+  sys.modules[labs_module_name] = sys.modules[new_module_name]
+  return sys.modules[labs_module_name]
+
+taskqueue = _map_module('taskqueue')
+taskqueue_service_pb = _map_module('taskqueue_service_pb')
+taskqueue_stub = _map_module('taskqueue_stub')
 
=== removed file 'AppServer/google/appengine/api/labs/taskqueue/taskqueue.py'
--- AppServer/google/appengine/api/labs/taskqueue/taskqueue.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/labs/taskqueue/taskqueue.py	1970-01-01 00:00:00 +0000
@@ -1,953 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2007 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Task Queue API.
-
-Enables an application to queue background work for itself. Work is done through
-webhooks that process tasks pushed from a queue. Tasks will execute in
-best-effort order of ETA. Webhooks that fail will cause tasks to be retried at a
-later time. Multiple queues may exist with independent throttling controls.
-
-Webhook URLs may be specified directly for Tasks, or the default URL scheme
-may be used, which will translate Task names into URLs relative to a Queue's
-base path. A default queue is also provided for simple usage.
-"""
-
-
-
-import calendar
-import datetime
-import os
-import re
-import time
-import urllib
-import urlparse
-
-import taskqueue_service_pb
-
-from google.appengine.api import apiproxy_stub_map
-from google.appengine.api import namespace_manager
-from google.appengine.api import urlfetch
-from google.appengine.runtime import apiproxy_errors
-import os,sys
-APPSCALE_HOME = os.environ.get("APPSCALE_HOME")
-
-
-class Error(Exception):
-  """Base-class for exceptions in this module."""
-
-
-class UnknownQueueError(Error):
-  """The queue specified is unknown."""
-
-
-class TransientError(Error):
-  """There was a transient error while accessing the queue.
-
-  Please Try again later.
-  """
-
-
-class InternalError(Error):
-  """There was an internal error while accessing this queue.
-
-  If this problem continues, please contact the App Engine team through
-  our support forum with a description of your problem.
-  """
-
-
-class InvalidTaskError(Error):
-  """The task's parameters, headers, or method is invalid."""
-
-
-class InvalidTaskNameError(InvalidTaskError):
-  """The task's name is invalid."""
-
-
-class TaskTooLargeError(InvalidTaskError):
-  """The task is too large with its headers and payload."""
-
-
-class TaskAlreadyExistsError(InvalidTaskError):
-  """Task already exists. It has not yet run."""
-
-
-class TombstonedTaskError(InvalidTaskError):
-  """Task has been tombstoned."""
-
-
-class InvalidUrlError(InvalidTaskError):
-  """The task's relative URL is invalid."""
-
-
-class BadTaskStateError(Error):
-  """The task is in the wrong state for the requested operation."""
-
-
-class InvalidQueueError(Error):
-  """The Queue's configuration is invalid."""
-
-
-class InvalidQueueNameError(InvalidQueueError):
-  """The Queue's name is invalid."""
-
-
-class _RelativeUrlError(Error):
-  """The relative URL supplied is invalid."""
-
-
-class PermissionDeniedError(Error):
-  """The requested operation is not allowed for this app."""
-
-
-class DuplicateTaskNameError(Error):
-  """The add arguments contain tasks with identical names."""
-
-
-class TooManyTasksError(Error):
-  """Too many tasks were present in a single function call."""
-
-
-class DatastoreError(Error):
-  """There was a datastore error while accessing the queue."""
-
-
-class BadTransactionStateError(Error):
-  """The state of the current transaction does not permit this operation."""
-
-BadTransactionState = BadTransactionStateError
-
-MAX_QUEUE_NAME_LENGTH = 100
-
-MAX_TASK_NAME_LENGTH = 500
-
-MAX_TASK_SIZE_BYTES = 10 * (2 ** 10)
-
-MAX_URL_LENGTH = 2083
-
-_DEFAULT_QUEUE = 'default'
-
-_DEFAULT_QUEUE_PATH = '/_ah/queue'
-
-_METHOD_MAP = {
-    'GET': taskqueue_service_pb.TaskQueueAddRequest.GET,
-    'POST': taskqueue_service_pb.TaskQueueAddRequest.POST,
-    'HEAD': taskqueue_service_pb.TaskQueueAddRequest.HEAD,
-    'PUT': taskqueue_service_pb.TaskQueueAddRequest.PUT,
-    'DELETE': taskqueue_service_pb.TaskQueueAddRequest.DELETE,
-}
-
-_NON_POST_METHODS = frozenset(['GET', 'HEAD', 'PUT', 'DELETE'])
-
-_BODY_METHODS = frozenset(['POST', 'PUT'])
-
-_TASK_NAME_PATTERN = r'^[a-zA-Z0-9-]{1,%s}$' % MAX_TASK_NAME_LENGTH
-
-_TASK_NAME_RE = re.compile(_TASK_NAME_PATTERN)
-
-_QUEUE_NAME_PATTERN = r'^[a-zA-Z0-9-]{1,%s}$' % MAX_QUEUE_NAME_LENGTH
-
-_QUEUE_NAME_RE = re.compile(_QUEUE_NAME_PATTERN)
-
-_ERROR_MAPPING = {
-    taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE: UnknownQueueError,
-    taskqueue_service_pb.TaskQueueServiceError.TRANSIENT_ERROR:
-        TransientError,
-    taskqueue_service_pb.TaskQueueServiceError.INTERNAL_ERROR: InternalError,
-    taskqueue_service_pb.TaskQueueServiceError.TASK_TOO_LARGE:
-        TaskTooLargeError,
-    taskqueue_service_pb.TaskQueueServiceError.INVALID_TASK_NAME:
-    InvalidTaskNameError,
-        taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_NAME:
-    InvalidQueueNameError,
-    taskqueue_service_pb.TaskQueueServiceError.INVALID_URL: InvalidUrlError,
-    taskqueue_service_pb.TaskQueueServiceError.INVALID_QUEUE_RATE:
-        InvalidQueueError,
-    taskqueue_service_pb.TaskQueueServiceError.PERMISSION_DENIED:
-        PermissionDeniedError,
-    taskqueue_service_pb.TaskQueueServiceError.TASK_ALREADY_EXISTS:
-        TaskAlreadyExistsError,
-    taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_TASK:
-        TombstonedTaskError,
-    taskqueue_service_pb.TaskQueueServiceError.INVALID_ETA: InvalidTaskError,
-    taskqueue_service_pb.TaskQueueServiceError.INVALID_REQUEST: Error,
-    taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_TASK: Error,
-    taskqueue_service_pb.TaskQueueServiceError.TOMBSTONED_QUEUE: Error,
-    taskqueue_service_pb.TaskQueueServiceError.DUPLICATE_TASK_NAME:
-        DuplicateTaskNameError,
-
-    taskqueue_service_pb.TaskQueueServiceError.TOO_MANY_TASKS:
-        TooManyTasksError,
-
-}
-
-_PRESERVE_ENVIRONMENT_HEADERS = (
-    ('X-AppEngine-Default-Namespace', 'HTTP_X_APPENGINE_DEFAULT_NAMESPACE'),)
-
-
-class _UTCTimeZone(datetime.tzinfo):
-  """UTC timezone."""
-
-  ZERO = datetime.timedelta(0)
-
-  def utcoffset(self, dt):
-    return self.ZERO
-
-  def dst(self, dt):
-    return self.ZERO
-
-  def tzname(self, dt):
-    return 'UTC'
-
-
-_UTC = _UTCTimeZone()
-
-
-def _parse_relative_url(relative_url):
-  """Parses a relative URL and splits it into its path and query string.
-
-  Args:
-    relative_url: The relative URL, starting with a '/'.
-
-  Returns:
-    Tuple (path, query) where:
-      path: The path in the relative URL.
-      query: The query string in the URL without the '?' character.
-
-  Raises:
-    _RelativeUrlError if the relative_url is invalid for whatever reason
-  """
-  if not relative_url:
-    raise _RelativeUrlError('Relative URL is empty')
-  (scheme, netloc, path, query, fragment) = urlparse.urlsplit(relative_url)
-  if scheme or netloc:
-    raise _RelativeUrlError('Relative URL may not have a scheme or location')
-  if fragment:
-    raise _RelativeUrlError('Relative URL may not specify a fragment')
-  if not path or path[0] != '/':
-    raise _RelativeUrlError('Relative URL path must start with "/"')
-  return path, query
-
-
-def _flatten_params(params):
-  """Converts a dictionary of parameters to a list of parameters.
-
-  Any unicode strings in keys or values will be encoded as UTF-8.
-
-  Args:
-    params: Dictionary mapping parameter keys to values. Values will be
-      converted to a string and added to the list as tuple (key, value). If
-      a values is iterable and not a string, each contained value will be
-      added as a separate (key, value) tuple.
-
-  Returns:
-    List of (key, value) tuples.
-  """
-  def get_string(value):
-    if isinstance(value, unicode):
-      return unicode(value).encode('utf-8')
-    else:
-      return str(value)
-
-  param_list = []
-  for key, value in params.iteritems():
-    key = get_string(key)
-    if isinstance(value, basestring):
-      param_list.append((key, get_string(value)))
-    else:
-      try:
-        iterator = iter(value)
-      except TypeError:
-        param_list.append((key, str(value)))
-      else:
-        param_list.extend((key, get_string(v)) for v in iterator)
-
-  return param_list
-
-
-class Task(object):
-  """Represents a single Task on a queue."""
-
-  __CONSTRUCTOR_KWARGS = frozenset([
-      'countdown', 'eta', 'headers', 'method', 'name', 'params', 'url'])
-
-  __eta_posix = None
-
-  def __init__(self, payload=None, **kwargs):
-    """Initializer.
-
-    All parameters are optional.
-
-    Args:
-      payload: The payload data for this Task that will be delivered to the
-        webhook as the HTTP request body. This is only allowed for POST and PUT
-        methods.
-      countdown: Time in seconds into the future that this Task should execute.
-        Defaults to zero.
-      eta: Absolute time when the Task should execute. May not be specified
-        if 'countdown' is also supplied. This may be timezone-aware or
-        timezone-naive.
-      headers: Dictionary of headers to pass to the webhook. Values in the
-        dictionary may be iterable to indicate repeated header fields.
-      method: Method to use when accessing the webhook. Defaults to 'POST'.
-      name: Name to give the Task; if not specified, a name will be
-        auto-generated when added to a queue and assigned to this object. Must
-        match the _TASK_NAME_PATTERN regular expression.
-      params: Dictionary of parameters to use for this Task. For POST requests
-        these params will be encoded as 'application/x-www-form-urlencoded' and
-        set to the payload. For all other methods, the parameters will be
-        converted to a query string. May not be specified if the URL already
-        contains a query string.
-      url: Relative URL where the webhook that should handle this task is
-        located for this application. May have a query string unless this is
-        a POST method.
-
-    Raises:
-      InvalidTaskError if any of the parameters are invalid;
-      InvalidTaskNameError if the task name is invalid; InvalidUrlError if
-      the task URL is invalid or too long; TaskTooLargeError if the task with
-      its payload is too large.
-    """
-    args_diff = set(kwargs.iterkeys()) - self.__CONSTRUCTOR_KWARGS
-    if args_diff:
-      raise TypeError('Invalid arguments: %s' % ', '.join(args_diff))
-
-    self.__name = kwargs.get('name')
-    if self.__name and not _TASK_NAME_RE.match(self.__name):
-      raise InvalidTaskNameError(
-          'Task name does not match expression "%s"; found %s' %
-          (_TASK_NAME_PATTERN, self.__name))
-
-    self.__default_url, self.__relative_url, query = Task.__determine_url(
-        kwargs.get('url', ''))
-    self.__headers = urlfetch._CaselessDict()
-    self.__headers.update(kwargs.get('headers', {}))
-    self.__method = kwargs.get('method', 'POST').upper()
-    self.__payload = None
-    params = kwargs.get('params', {})
-
-    for header_name, environ_name in _PRESERVE_ENVIRONMENT_HEADERS:
-      value = os.environ.get(environ_name)
-      if value is not None:
-        self.__headers.setdefault(header_name, value)
-
-    self.__headers.setdefault('X-AppEngine-Current-Namespace',
-                              namespace_manager.get_namespace())
-    if query and params:
-      raise InvalidTaskError('Query string and parameters both present; '
-                             'only one of these may be supplied')
-
-    if self.__method == 'POST':
-      if payload and params:
-        raise InvalidTaskError('Message body and parameters both present for '
-                               'POST method; only one of these may be supplied')
-      elif query:
-        raise InvalidTaskError('POST method may not have a query string; '
-                               'use the "params" keyword argument instead')
-      elif params:
-        self.__payload = Task.__encode_params(params)
-        self.__headers.setdefault(
-            'content-type', 'application/x-www-form-urlencoded')
-      elif payload is not None:
-        self.__payload = Task.__convert_payload(payload, self.__headers)
-    elif self.__method in _NON_POST_METHODS:
-      if payload and self.__method not in _BODY_METHODS:
-        raise InvalidTaskError('Payload may only be specified for methods %s' %
-                               ', '.join(_BODY_METHODS))
-      if payload:
-        self.__payload = Task.__convert_payload(payload, self.__headers)
-      if params:
-        query = Task.__encode_params(params)
-      if query:
-        self.__relative_url = '%s?%s' % (self.__relative_url, query)
-    else:
-      raise InvalidTaskError('Invalid method: %s' % self.__method)
-
-    self.__headers_list = _flatten_params(self.__headers)
-    self.__eta_posix = Task.__determine_eta_posix(
-        kwargs.get('eta'), kwargs.get('countdown'))
-    self.__eta = None
-    self.__enqueued = False
-
-    if self.size > MAX_TASK_SIZE_BYTES:
-      raise TaskTooLargeError('Task size must be less than %d; found %d' %
-                              (MAX_TASK_SIZE_BYTES, self.size))
-
-  @staticmethod
-  def __determine_url(relative_url):
-    """Determines the URL of a task given a relative URL and a name.
-
-    Args:
-      relative_url: The relative URL for the Task.
-
-    Returns:
-      Tuple (default_url, relative_url, query) where:
-        default_url: True if this Task is using the default URL scheme;
-          False otherwise.
-        relative_url: String containing the relative URL for this Task.
-        query: The query string for this task.
-
-    Raises:
-      InvalidUrlError if the relative_url is invalid.
-    """
-    if not relative_url:
-      default_url, query = True, ''
-    else:
-      default_url = False
-      try:
-        relative_url, query = _parse_relative_url(relative_url)
-      except _RelativeUrlError, e:
-        raise InvalidUrlError(e)
-
-    if len(relative_url) > MAX_URL_LENGTH:
-      raise InvalidUrlError(
-          'Task URL must be less than %d characters; found %d' %
-          (MAX_URL_LENGTH, len(relative_url)))
-
-    return (default_url, relative_url, query)
-
-  @staticmethod
-  def __determine_eta_posix(eta=None, countdown=None, current_time=time.time):
-    """Determines the ETA for a task.
-
-    If 'eta' and 'countdown' are both None, the current time will be used.
-    Otherwise, only one of them may be specified.
-
-    Args:
-      eta: A datetime.datetime specifying the absolute ETA or None;
-        this may be timezone-aware or timezone-naive.
-      countdown: Count in seconds into the future from the present time that
-        the ETA should be assigned to.
-
-    Returns:
-      A float giving a POSIX timestamp containing the ETA.
-
-    Raises:
-      InvalidTaskError if the parameters are invalid.
-    """
-    if eta is not None and countdown is not None:
-      raise InvalidTaskError('May not use a countdown and ETA together')
-    elif eta is not None:
-      if not isinstance(eta, datetime.datetime):
-        raise InvalidTaskError('ETA must be a datetime.datetime instance')
-      elif eta.tzinfo is None:
-        return time.mktime(eta.timetuple()) + eta.microsecond*1e-6
-      else:
-        return calendar.timegm(eta.utctimetuple()) + eta.microsecond*1e-6
-    elif countdown is not None:
-      try:
-        countdown = float(countdown)
-      except ValueError:
-        raise InvalidTaskError('Countdown must be a number')
-      except OverflowError:
-        raise InvalidTaskError('Countdown out of range')
-      else:
-        return current_time() + countdown
-    else:
-      return current_time()
-
-  @staticmethod
-  def __encode_params(params):
-    """URL-encodes a list of parameters.
-
-    Args:
-      params: Dictionary of parameters, possibly with iterable values.
-
-    Returns:
-      URL-encoded version of the params, ready to be added to a query string or
-      POST body.
-    """
-    return urllib.urlencode(_flatten_params(params))
-
-  @staticmethod
-  def __convert_payload(payload, headers):
-    """Converts a Task payload into UTF-8 and sets headers if necessary.
-
-    Args:
-      payload: The payload data to convert.
-      headers: Dictionary of headers.
-
-    Returns:
-      The payload as a non-unicode string.
-
-    Raises:
-      InvalidTaskError if the payload is not a string or unicode instance.
-    """
-    if isinstance(payload, unicode):
-      headers.setdefault('content-type', 'text/plain; charset=utf-8')
-      payload = payload.encode('utf-8')
-    elif not isinstance(payload, str):
-      raise InvalidTaskError(
-          'Task payloads must be strings; invalid payload: %r' % payload)
-    return payload
-
-  @property
-  def on_queue_url(self):
-    """Returns True if this Task will run on the queue's URL."""
-    return self.__default_url
-
-  @property
-  def eta_posix(self):
-    """Returns a POSIX timestamp giving when this Task will execute."""
-    if self.__eta_posix is None and self.__eta is not None:
-      self.__eta_posix = Task.__determine_eta_posix(self.__eta)
-    return self.__eta_posix
-
-  @property
-  def eta(self):
-    """Returns a datetime when this Task will execute."""
-    if self.__eta is None and self.__eta_posix is not None:
-      self.__eta = datetime.datetime.fromtimestamp(self.__eta_posix, _UTC)
-    return self.__eta
-
-  @property
-  def headers(self):
-    """Returns a copy of the headers for this Task."""
-    return self.__headers.copy()
-
-  @property
-  def method(self):
-    """Returns the method to use for this Task."""
-    return self.__method
-
-  @property
-  def name(self):
-    """Returns the name of this Task.
-
-    Will be None if using auto-assigned Task names and this Task has not yet
-    been added to a Queue.
-    """
-    return self.__name
-
-  @property
-  def payload(self):
-    """Returns the payload for this task, which may be None."""
-    return self.__payload
-
-  @property
-  def size(self):
-    """Returns the size of this task in bytes."""
-    HEADER_SEPERATOR = len(': \r\n')
-    header_size = sum((len(key) + len(value) + HEADER_SEPERATOR)
-                      for key, value in self.__headers_list)
-    return (len(self.__method) + len(self.__payload or '') +
-            len(self.__relative_url) + header_size)
-
-  @property
-  def url(self):
-    """Returns the relative URL for this Task."""
-    return self.__relative_url
-
-  @property
-  def was_enqueued(self):
-    """Returns True if this Task has been enqueued.
-
-    Note: This will not check if this task already exists in the queue.
-    """
-    return self.__enqueued
-
-  def add(self, queue_name=_DEFAULT_QUEUE, transactional=False):
-    """Adds this Task to a queue. See Queue.add."""
-    return Queue(queue_name).add(self, transactional=transactional)
-
-
-class Queue(object):
-  """Represents a Queue."""
-
-  def __init__(self, name=_DEFAULT_QUEUE):
-    """Initializer.
-
-    Args:
-      name: Name of this queue. If not supplied, defaults to the default queue.
-
-    Raises:
-      InvalidQueueNameError if the queue name is invalid.
-    """
-    if not _QUEUE_NAME_RE.match(name):
-      raise InvalidQueueNameError(
-          'Queue name does not match pattern "%s"; found %s' %
-          (_QUEUE_NAME_PATTERN, name))
-    self.__name = name
-    self.__url = '%s/%s' % (_DEFAULT_QUEUE_PATH, self.__name)
-
-    self._app = None
-
-  def add(self, task, transactional=False):
-    """Adds a Task or list of Tasks to this Queue.
-
-    If a list of more than one Tasks is given, a raised exception does not
-    guarantee that no tasks were added to the queue (unless transactional is set
-    to True). To determine which tasks were successfully added when an exception
-    is raised, check the Task.was_enqueued property.
-
-    Args:
-      task: A Task instance or a list of Task instances that will added to the
-        queue.
-      transactional: If False adds the Task(s) to a queue irrespectively to the
-        enclosing transaction success or failure. An exception is raised if True
-        and called outside of a transaction. (optional)
-
-    Returns:
-      The Task or list of tasks that was supplied to this method.
-
-    Raises:
-      BadTaskStateError: if the Task(s) has already been added to a queue.
-      BadTransactionStateError: if the transactional argument is true but this
-        call is being made outside of the context of a transaction.
-      Error-subclass on application errors.
-    """
-    try:
-      tasks = list(iter(task))
-    except TypeError:
-      tasks = [task]
-      multiple = False
-    else:
-      multiple = True
-
-    self.__AddTasks(tasks, transactional)
-
-    if multiple:
-      return tasks
-    else:
-      assert len(tasks) == 1
-      return tasks[0]
-
-  def __AddTasks(self, tasks, transactional):
-    """Internal implementation of .add() where tasks must be a list."""
-
-    request = taskqueue_service_pb.TaskQueueBulkAddRequest()
-    response = taskqueue_service_pb.TaskQueueBulkAddResponse()
-
-    task_names = set()
-    for task in tasks:
-      if task.name:
-        if task.name in task_names:
-          raise DuplicateTaskNameError(
-              'The task name %r is used more than once in the request' %
-              task.name)
-        task_names.add(task.name)
-
-      self.__FillAddRequest(task, request.add_add_request(), transactional)
-
-    try:
-      apiproxy_stub_map.MakeSyncCall('taskqueue', 'BulkAdd', request, response)
-    except apiproxy_errors.ApplicationError, e:
-      raise self.__TranslateError(e.application_error, e.error_detail)
-
-    assert response.taskresult_size() == len(tasks), (
-        'expected %d results from BulkAdd(), got %d' % (
-            len(tasks), response.taskresult_size()))
-
-    exception = None
-    for task, task_result in zip(tasks, response.taskresult_list()):
-      if task_result.result() == taskqueue_service_pb.TaskQueueServiceError.OK:
-        if task_result.has_chosen_task_name():
-          task._Task__name = task_result.chosen_task_name()
-        task._Task__enqueued = True
-      elif (task_result.result() ==
-            taskqueue_service_pb.TaskQueueServiceError.SKIPPED):
-        pass
-      elif exception is None:
-        exception = self.__TranslateError(task_result.result())
-
-    if exception is not None:
-      raise exception
-
-    return tasks
-
-  def __FillAddRequest(self, task, task_request, transactional):
-    """Populates a TaskQueueAddRequest with the data from a Task instance.
-
-    Args:
-      task: The Task instance to use as a source for the data to be added to
-        task_request.
-      task_request: The taskqueue_service_pb.TaskQueueAddRequest to populate.
-      transactional: If true then populates the task_request.transaction message
-        with information from the enclosing transaction (if any).
-
-    Raises:
-      BadTaskStateError: If the task was already added to a Queue.
-      BadTransactionStateError: If the transactional argument is True and there
-        is no enclosing transaction.
-      InvalidTaskNameError: If the transactional argument is True and the task
-        is named.
-    """
-    if task.was_enqueued:
-      raise BadTaskStateError('Task has already been enqueued')
-
-    adjusted_url = task.url
-    if task.on_queue_url:
-      adjusted_url = self.__url + task.url
-
-
-    task_request.set_queue_name(self.__name)
-    task_request.set_eta_usec(long(task.eta_posix * 1e6))
-    task_request.set_method(_METHOD_MAP.get(task.method))
-    task_request.set_url(adjusted_url)
-
-    if task.name:
-      task_request.set_task_name(task.name)
-    else:
-      task_request.set_task_name('')
-
-    if task.payload:
-      task_request.set_body(task.payload)
-    for key, value in _flatten_params(task.headers):
-      header = task_request.add_header()
-      header.set_key(key)
-      header.set_value(value)
-
-    if self._app:
-      task_request.set_app_id(self._app)
-
-    if transactional:
-      from google.appengine.api import datastore
-      if not datastore._MaybeSetupTransaction(task_request, []):
-        raise BadTransactionStateError(
-            'Transactional adds are not allowed outside of transactions')
-
-    if task_request.has_transaction() and task.name:
-      raise InvalidTaskNameError(
-          'Task bound to a transaction cannot be named.')
-
-  @property
-  def name(self):
-    """Returns the name of this queue."""
-    return self.__name
-
-  @staticmethod
-  def __TranslateError(error, detail=''):
-    """Translates a TaskQueueServiceError into an exception.
-
-    Args:
-      error: Value from TaskQueueServiceError enum.
-      detail: A human-readable description of the error.
-
-    Returns:
-      The corresponding Exception sub-class for that error code.
-    """
-    if (error >= taskqueue_service_pb.TaskQueueServiceError.DATASTORE_ERROR
-        and isinstance(error, int)):
-      from google.appengine.api import datastore
-      datastore_exception = datastore._DatastoreExceptionFromErrorCodeAndDetail(
-          error - taskqueue_service_pb.TaskQueueServiceError.DATASTORE_ERROR,
-          detail)
-
-      class JointException(datastore_exception.__class__, DatastoreError):
-        """There was a datastore error while accessing the queue."""
-        __msg = (u'taskqueue.DatastoreError caused by: %s %s' %
-                 (datastore_exception.__class__, detail))
-        def __str__(self):
-          return JointException.__msg
-
-      return JointException()
-    else:
-      exception_class = _ERROR_MAPPING.get(error, None)
-      if exception_class:
-        return exception_class(detail)
-      else:
-        return Error('Application error %s: %s' % (error, detail))
-
-
-def add(*args, **kwargs):
-  """Convenience method will create a Task and add it to a queue.
-
-  All parameters are optional.
-
-  Args:
-    name: Name to give the Task; if not specified, a name will be
-      auto-generated when added to a queue and assigned to this object. Must
-      match the _TASK_NAME_PATTERN regular expression.
-    queue_name: Name of this queue. If not supplied, defaults to
-      the default queue.
-    url: Relative URL where the webhook that should handle this task is
-      located for this application. May have a query string unless this is
-      a POST method.
-    method: Method to use when accessing the webhook. Defaults to 'POST'.
-    headers: Dictionary of headers to pass to the webhook. Values in the
-      dictionary may be iterable to indicate repeated header fields.
-    payload: The payload data for this Task that will be delivered to the
-      webhook as the HTTP request body. This is only allowed for POST and PUT
-      methods.
-    params: Dictionary of parameters to use for this Task. For POST requests
-      these params will be encoded as 'application/x-www-form-urlencoded' and
-      set to the payload. For all other methods, the parameters will be
-      converted to a query string. May not be specified if the URL already
-      contains a query string.
-    transactional: If False adds the Task(s) to a queue irrespectively to the
-      enclosing transaction success or failure. An exception is raised if True
-      and called outside of a transaction. (optional)
-    countdown: Time in seconds into the future that this Task should execute.
-      Defaults to zero.
-    eta: Absolute time when the Task should execute. May not be specified
-      if 'countdown' is also supplied. This may be timezone-aware or
-      timezone-naive.
-
-  Returns:
-    The Task that was added to the queue.
-
-  Raises:
-      InvalidTaskError if any of the parameters are invalid;
-      InvalidTaskNameError if the task name is invalid; InvalidUrlError if
-      the task URL is invalid or too long; TaskTooLargeError if the task with
-      its payload is too large.
-  """
-  transactional = kwargs.pop('transactional', False)
-  queue_name = kwargs.pop('queue_name', _DEFAULT_QUEUE)
-  return Task(*args, **kwargs).add(
-      queue_name=queue_name, transactional=transactional)
-
-def getLang(file):
-  supportedExtensions = {
-    "rb" : "ruby",
-    "py" : "python",
-    "pl" : "perl",
-  }
-
-  # return None if file is None
-  extension = file.split(".")[-1]
-
-  try:
-    lang = supportedExtensions[extension]
-    return lang
-  except:
-    sys.stderr.write("extension " + extension + " not recognized\n")
-    return "none"
-
-def writeTempFile(suffix, data):
-  suffix = urllib.unquote(suffix)
-  regex = r"[^\w\d/\.-]"
-  pattern = re.compile(regex)
-  suffix = pattern.sub('', suffix)
-
-  fileLoc = "/tmp/" + suffix
-  f = open(fileLoc, "w+")
-  f.write(data)
-  f.close()
-  return fileLoc
-
-def getAllIPs():
-  all_ips = []
-  fileLoc = APPSCALE_HOME + "/.appscale/all_ips"
-  if os.path.exists(fileLoc):
-    f = open(fileLoc)
-    text = f.read()
-
-def getNumOfNodes():
-  num_of_nodes = 0
-  fileLoc = APPSCALE_HOME + "/.appscale/num_of_nodes"
-  if os.path.exists(fileLoc):
-    f = open(fileLoc)
-    num_of_nodes = int(f.read())
-
-  return num_of_nodes
-
-def putMRInput(data, inputLoc):
-  inputLoc = urllib.unquote(inputLoc)
-  regex = r"[^\w\d/\.-]"
-  pattern = re.compile(regex)
-  inputLoc = pattern.sub('', inputLoc)
-
-  fileLoc = "/tmp/" + inputLoc
-  f = open(fileLoc, "w+")
-  f.write(data)
-  f.close()
-
-  removeInput = APPSCALE_HOME + "/AppDB/hadoop-0.20.0/bin/hadoop fs -rmr " + inputLoc
-  sys.stderr.write(removeInput + "\n")
-  os.system(removeInput)
-
-  put = APPSCALE_HOME + "/AppDB/hadoop-0.20.0/bin/hadoop fs -put " + fileLoc + " " + inputLoc
-  os.system(put)
-
-  return
-def runMRJob(mapper, reducer, inputLoc, outputLoc, config={}):
-  mapper = urllib.unquote(mapper)
-  reducer = urllib.unquote(reducer)
-  inputLoc = urllib.unquote(inputLoc)
-  outputLoc = urllib.unquote(outputLoc)
-
-  regex = r"[^\w\d/\.-]"
-  pattern = re.compile(regex)
-
-  mydir = os.getcwd() + "/"
-  mapper = "\"" + getLang(mapper) + " " + mydir + pattern.sub('', mapper) + "\""
-  reducer = "\"" + getLang(reducer) + " " + mydir + pattern.sub('', reducer) + "\""
-  inputLoc = pattern.sub('', inputLoc)
-  outputLoc = pattern.sub('', outputLoc)
-
-  removeOutput = APPSCALE_HOME + "/AppDB/hadoop-0.20.0/bin/hadoop fs -rmr " + outputLoc
-  sys.stderr.write(removeOutput + "\n")
-  os.system(removeOutput)
-
-  formattedConfig = ""
-  for key in config:
-    formattedConfig = formattedConfig + " -D " + key + "=" + config[key]
-
-  command = APPSCALE_HOME + "/AppDB/hadoop-0.20.0/bin/hadoop jar " + APPSCALE_HOME + "/AppDB/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar " + formattedConfig + " -input " + inputLoc + " -output " + outputLoc + " -mapper " + mapper + " -reducer " + reducer
-  sys.stderr.write("\n" + command + "\n")
-  start = time.time()
-  os.system(command)
-  end = time.time()
-  sys.stderr.write("\nTime elapsed = " + str(end - start) + "seconds\n")
-
-def getMROutput(outputLoc):
-  outputLoc = urllib.unquote(outputLoc)
-  regex = r"[^\w\d/\.-]"
-  pattern = re.compile(regex)
-  outputLoc = pattern.sub('', outputLoc)
-
-  fileLoc = "/tmp/" + outputLoc
-
-  rmr = "rm -rf " + fileLoc
-  os.system(rmr)
-  get = APPSCALE_HOME + "/AppDB/hadoop-0.20.0/bin/hadoop fs -get " + outputLoc + " " + fileLoc
-  os.system(get)
-
-  contents = "no output"
-  if os.path.exists(fileLoc):
-    cmd = "cat " + fileLoc + "/part*"
-    contents = os.popen(cmd).read()
-
-
-  sys.stderr.write(contents)
-  return contents
-
-def getMRLogs(outputLoc):
-  outputLoc = urllib.unquote(outputLoc)
-  regex = r"[^\w\d/\.-]"
-  pattern = re.compile(regex)
-  outputLoc = pattern.sub('', outputLoc)
-
-  fileLoc = "/tmp/" + outputLoc
-
-  rmr = "rm -rf " + fileLoc
-  os.system(rmr)
-
-  get = APPSCALE_HOME + "/AppDB/hadoop-0.20.0/bin/hadoop fs -get " + outputLoc + " " + fileLoc
-  os.system(get)
-
-  contents = "no logs"
-  if os.path.exists(fileLoc):
-    cmd = "cat " + fileLoc + "/_logs/history/*"
-    contents = os.popen(cmd).read()
-
-
-  sys.stderr.write(contents)
-  return contents
-
-
 
=== removed file 'AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py'
--- AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py	2010-11-30 10:37:25 +0000
+++ AppServer/google/appengine/api/labs/taskqueue/taskqueue_service_pb.py	1970-01-01 00:00:00 +0000
@@ -1,5229 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2007 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from google.net.proto import ProtocolBuffer
-import array
-import dummy_thread as thread
-
-__pychecker__ = """maxreturns=0 maxbranches=0 no-callinit
-                   unusednames=printElemNumber,debug_strs no-special"""
-
-from google.appengine.datastore.datastore_v3_pb import *
-import google.appengine.datastore.datastore_v3_pb
-from google.net.proto.message_set import MessageSet
-class TaskQueueServiceError(ProtocolBuffer.ProtocolMessage):
-
-  OK           =    0
-  UNKNOWN_QUEUE =    1
-  TRANSIENT_ERROR =    2
-  INTERNAL_ERROR =    3
-  TASK_TOO_LARGE =    4
-  INVALID_TASK_NAME =    5
-  INVALID_QUEUE_NAME =    6
-  INVALID_URL  =    7
-  INVALID_QUEUE_RATE =    8
-  PERMISSION_DENIED =    9
-  TASK_ALREADY_EXISTS =   10
-  TOMBSTONED_TASK =   11
-  INVALID_ETA  =   12
-  INVALID_REQUEST =   13
-  UNKNOWN_TASK =   14
-  TOMBSTONED_QUEUE =   15
-  DUPLICATE_TASK_NAME =   16
-  SKIPPED      =   17
-  TOO_MANY_TASKS =   18
-  INVALID_PAYLOAD =   19
-  INVALID_RETRY_PARAMETERS =   20
-  DATASTORE_ERROR = 10000
-
-  _ErrorCode_NAMES = {
-    0: "OK",
-    1: "UNKNOWN_QUEUE",
-    2: "TRANSIENT_ERROR",
-    3: "INTERNAL_ERROR",
-    4: "TASK_TOO_LARGE",
-    5: "INVALID_TASK_NAME",
-    6: "INVALID_QUEUE_NAME",
-    7: "INVALID_URL",
-    8: "INVALID_QUEUE_RATE",
-    9: "PERMISSION_DENIED",
-    10: "TASK_ALREADY_EXISTS",
-    11: "TOMBSTONED_TASK",
-    12: "INVALID_ETA",
-    13: "INVALID_REQUEST",
-    14: "UNKNOWN_TASK",
-    15: "TOMBSTONED_QUEUE",
-    16: "DUPLICATE_TASK_NAME",
-    17: "SKIPPED",
-    18: "TOO_MANY_TASKS",
-    19: "INVALID_PAYLOAD",
-    20: "INVALID_RETRY_PARAMETERS",
-    10000: "DATASTORE_ERROR",
-  }
-
-  def ErrorCode_Name(cls, x): return cls._ErrorCode_NAMES.get(x, "")
-  ErrorCode_Name = classmethod(ErrorCode_Name)
-
-
-  def __init__(self, contents=None):
-    pass
-    if contents is not None: self.MergeFromString(contents)
-
-
-  def MergeFrom(self, x):
-    assert x is not self
-
-  def Equals(self, x):
-    if x is self: return 1
-    return 1
-
-  def IsInitialized(self, debug_strs=None):
-    initialized = 1
-    return initialized
-
-  def ByteSize(self):
-    n = 0
-    return n + 0
-
-  def Clear(self):
-    pass
-
-  def OutputUnchecked(self, out):
-    pass
-
-  def TryMerge(self, d):
-    while d.avail() > 0:
-      tt = d.getVarInt32()
-      if (tt == 0): raise ProtocolBuffer.ProtocolBufferDecodeError
-      d.skipData(tt)
-
-
-  def __str__(self, prefix="", printElemNumber=0):
-    res=""
-    return res
-
-
-  def _BuildTagLookupTable(sparse, maxtag, default=None):
-    return tuple([sparse.get(i, default) for i in xrange(0, 1+maxtag)])
-
-
-  _TEXT = _BuildTagLookupTable({
-    0: "ErrorCode",
-  }, 0)
-
-  _TYPES = _BuildTagLookupTable({
-    0: ProtocolBuffer.Encoder.NUMERIC,
-  }, 0, ProtocolBuffer.Encoder.MAX_TYPE)
-
-  _STYLE = """"""
-  _STYLE_CONTENT_TYPE = """"""
-class TaskQueueRetryParameters(ProtocolBuffer.ProtocolMessage):
-  has_retry_limit_ = 0
-  retry_limit_ = 0
-  has_age_limit_sec_ = 0
-  age_limit_sec_ = 0
-  has_min_backoff_sec_ = 0
-  min_backoff_sec_ = 0.1
-  has_max_backoff_sec_ = 0
-  max_backoff_sec_ = 3600.0
-  has_max_doublings_ = 0
-  max_doublings_ = 16
-
-  def __init__(self, contents=None):
-    if contents is not None: self.MergeFromString(contents)
-
-  def retry_limit(self): return self.retry_limit_
-
-  def set_retry_limit(self, x):
-    self.has_retry_limit_ = 1
-    self.retry_limit_ = x
-
-  def clear_retry_limit(self):
-    if self.has_retry_limit_:
-      self.has_retry_limit_ = 0
-      self.retry_limit_ = 0
-
-  def has_retry_limit(self): return self.has_retry_limit_
-
-  def age_limit_sec(self): return self.age_limit_sec_
-
-  def set_age_limit_sec(self, x):
-    self.has_age_limit_sec_ = 1
-    self.age_limit_sec_ = x
-
-  def clear_age_limit_sec(self):
-    if self.has_age_limit_sec_:
-      self.has_age_limit_sec_ = 0
-      self.age_limit_sec_ = 0
-
-  def has_age_limit_sec(self): return self.has_age_limit_sec_
-
-  def min_backoff_sec(self): return self.min_backoff_sec_
-
-  def set_min_backoff_sec(self, x):
-    self.has_min_backoff_sec_ = 1
-    self.min_backoff_sec_ = x
-
-  def clear_min_backoff_sec(self):
-    if self.has_min_backoff_sec_:
-      self.has_min_backoff_sec_ = 0
-      self.min_backoff_sec_ = 0.1
-
-  def has_min_backoff_sec(self): return self.has_min_backoff_sec_
-
-  def max_backoff_sec(self): return self.max_backoff_sec_
-
-  def set_max_backoff_sec(self, x):
-    self.has_max_backoff_sec_ = 1
-    self.max_backoff_sec_ = x
-
-  def clear_max_backoff_sec(self):
-    if self.has_max_backoff_sec_:
-      self.has_max_backoff_sec_ = 0
-      self.max_backoff_sec_ = 3600.0
-
-  def has_max_backoff_sec(self): return self.has_max_backoff_sec_
-
-  def max_doublings(self): return self.max_doublings_
-
-  def set_max_doublings(self, x):
-    self.has_max_doublings_ = 1
-    self.max_doublings_ = x
-
-  def clear_max_doublings(self):
-    if self.has_max_doublings_:
-      self.has_max_doublings_ = 0
-      self.max_doublings_ = 16
-
-  def has_max_doublings(self): return self.has_max_doublings_
-
-
-  def MergeFrom(self, x):
-    assert x is not self
-    if (x.has_retry_limit()): self.set_retry_limit(x.retry_limit())
-    if (x.has_age_limit_sec()): self.set_age_limit_sec(x.age_limit_sec())
-    if (x.has_min_backoff_sec()): self.set_min_backoff_sec(x.min_backoff_sec())
-    if (x.has_max_backoff_sec()): self.set_max_backoff_sec(x.max_backoff_sec())
-    if (x.has_max_doublings()): self.set_max_doublings(x.max_doublings())
-
-  def Equals(self, x):
-    if x is self: return 1
-    if self.has_retry_limit_ != x.has_retry_limit_: return 0
-    if self.has_retry_limit_ and self.retry_limit_ != x.retry_limit_: return 0
-    if self.has_age_limit_sec_ != x.has_age_limit_sec_: return 0
-    if self.has_age_limit_sec_ and self.age_limit_sec_ != x.age_limit_sec_: return 0
-    if self.has_min_backoff_sec_ != x.has_min_backoff_sec_: return 0
-    if self.has_min_backoff_sec_ and self.min_backoff_sec_ != x.min_backoff_sec_: return 0
-    if self.has_max_backoff_sec_ != x.has_max_backoff_sec_: return 0
-    if self.has_max_backoff_sec_ and self.max_backoff_sec_ != x.max_backoff_sec_: return 0
-    if self.has_max_doublings_ != x.has_max_doublings_: return 0
-    if self.has_max_doublings_ and self.max_doublings_ != x.max_doublings_: return 0
-    return 1
-
-  def IsInitialized(self, debug_strs=None):