Merge lp:~cuteyw/drizzle/cloud-storage-engine-using-json-row-buffer-format into lp:~drizzle-trunk/drizzle/development

[Wei Ye]

[lp:~cuteyw/drizzle/cloud-storage-engine-using-json-row-buffer-format]

1696. By cactus <cactus@cactus-desktop> on 2010-08-19

merge

Unmerged revisions

1696. By cactus <cactus@cactus-desktop> on 2010-08-19

merge

1695. By cactus <cactus@cactus-desktop> on 2010-08-16

update header of ha_cloud.cc

1694. By cactus <cactus@cactus-desktop> on 2010-08-16

change row buffer fomat from csv to json, and now support nullable column

1693. By cactus <cactus@cactus-desktop> on 2010-08-13

AWSS3Service can send and receive binary data now

1692. By cactus <cactus@cactus-desktop> on 2010-08-12

adjust to lock api

1691. By cactus <cactus@cactus-desktop> on 2010-08-11

add update test

1690. By cactus <cactus@cactus-desktop> on 2010-08-10

merge

1689. By cactus <cactus@cactus-desktop> on 2010-08-10

update test

1688. By cactus <cactus@cactus-desktop> on 2010-08-10

update test

1687. By cactus <cactus@cactus-desktop> on 2010-08-10

update test result

=== added directory 'plugin/cloud_storage'
=== added file 'plugin/cloud_storage/awss3_service.cc'
--- plugin/cloud_storage/awss3_service.cc	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/awss3_service.cc	2010-08-19 16:21:41 +0000
@@ -0,0 +1,451 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "config.h"
+
+#include <cassert>
+#include <cstring>
+#include <iostream>
+
+#include <drizzled/algorithm/sha1.h>
+#include <gcrypt.h>
+
+#include "curl.h"
+#include "base64.h"
+#include "awss3_service.h"
+
+AWSS3Service::AWSS3Service(const std::string& bkt) :
+  CloudStorageService(bkt), connection(NULL)
+{
+  const std::string& host= std::string("http://") + bucket + "." + serviceURL();
+  connection= HTTP::ConnectionFactory::openHttpConnection(host);
+}
+
+AWSS3Service::~AWSS3Service()
+{
+  assert(connection);
+  HTTP::ConnectionFactory::destroyHttpConnection(connection);
+}
+
+void AWSS3Service::parseItemKey(BucketInfo* bucket_info, 
+                                xmlDocPtr doc,
+                                xmlNodePtr cur)
+{
+  xmlChar *str;
+  cur= cur->xmlChildrenNode;
+
+  while (cur != NULL)
+  {
+    if ((!xmlStrcmp(cur->name, (const xmlChar *) "Key")))
+    {
+      str= xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
+      bucket_info->keys.push_back((char *) str);
+      //TODO: save itemkey_z as well
+      xmlFree(str);
+    }
+    //TODO: save/cache LastModified, ETag, Size, Owner, etc
+    cur= cur->next;
+  }
+
+  return;
+}
+
+BucketInfo* AWSS3Service::parseBucketInfo(xmlDocPtr doc)
+{
+  xmlNodePtr cur;
+  xmlChar *str;
+
+  cur= xmlDocGetRootElement(doc);
+
+  if (cur == NULL)
+  {
+    // empty document
+    xmlFreeDoc(doc);
+    return NULL;
+  }
+  if (xmlStrcmp(cur->name, (const xmlChar *) "ListBucketResult"))
+  {
+    // document of the wrong type
+    xmlFreeDoc(doc);
+    return NULL;
+  }
+
+  BucketInfo* bucket_info= new BucketInfo();
+
+  cur= cur->xmlChildrenNode;
+  while (cur != NULL)
+  {
+    if ((!xmlStrcmp(cur->name, (const xmlChar *) "IsTruncated")))
+    {
+      str= xmlNodeListGetString(doc, cur->xmlChildrenNode, 1);
+      bucket_info->isTrucated= (strcasecmp("true", (char*) str) == 0);
+      xmlFree(str);
+    }
+    else if ((!xmlStrcmp(cur->name, (const xmlChar *) "Contents")))
+    {
+      parseItemKey(bucket_info, doc, cur);
+    }
+    cur= cur->next;
+  }
+
+  return bucket_info;
+}
+
+BucketInfo* AWSS3Service::parseBucketInfo(const std::string &content)
+{
+  xmlDocPtr doc= xmlReadMemory(content.c_str(), content.length(), "list.xml",
+      NULL, 0);
+  if (!doc)
+  {
+    return NULL;
+  }
+
+  return parseBucketInfo(doc);
+}
+
+std::string* AWSS3Service::makeAWSS3CanonicalString(
+    std::string *gp, const char *http_verb,
+    AWSS3Service::HTTPHeaderMap& headers, const char *aws_s3_objectname)
+{
+  const char *aws_s3_bucket= bucket.c_str();
+
+  gp->clear();
+
+  /* the HTTP method verb */
+  gp->append(http_verb);
+  gp->append("\n");
+
+  {
+    /* the Content-Md5, from the headers, if known */
+    HTTPHeaderMap::const_iterator iter= headers.find("Content-MD5");
+    if (iter != headers.end())
+    {
+      const std::string& md5= iter->second;
+      gp->append(md5);
+    }
+    gp->append("\n");
+  }
+
+  {
+    /* the Content-Type, from the headers, if known */
+    HTTPHeaderMap::const_iterator iter= headers.find("Content-Type");
+    if (iter != headers.end())
+    {
+      const std::string& type= iter->second;
+      gp->append(type);
+    }
+    gp->append("\n");
+  }
+
+  {
+    /* The Date header, if set.  Otherwise create one */
+    HTTPHeaderMap::const_iterator iter= headers.find("Date");
+    assert(iter != headers.end());
+    const std::string& date= iter->second;
+    gp->append(date);
+    gp->append("\n");
+  }
+
+  // TODO process x-amz headers
+
+  gp->append("/");
+  if (aws_s3_bucket)
+  {
+    gp->append(aws_s3_bucket);
+    if (aws_s3_objectname)
+    {
+      gp->append("/");
+      gp->append(aws_s3_objectname);
+    }
+  }
+
+  return gp;
+}
+
+std::string* AWSS3Service::makeAWSS3Authorization(
+    std::string *returnstr, const char *http_verb,
+    HTTPHeaderMap& headers, const char *aws_s3_objectname)
+{
+  const char *aws_access_key_id= keyID.c_str();
+  const char *aws_access_key_secret= keyValue.c_str();
+
+  std::string g;
+  std::string *gp;
+
+  if ((aws_access_key_id == NULL) || (aws_access_key_id[0] == '\0')
+      || (aws_access_key_secret == NULL) 
+      || (aws_access_key_secret[0] == '\0')
+      || (http_verb == NULL) || (http_verb[0] == '\0'))
+  {
+    returnstr->clear();
+    return returnstr;
+  }
+
+  gp= makeAWSS3CanonicalString(&g, http_verb, headers, aws_s3_objectname);
+
+  // calculate authorization signature value
+  gcry_control (GCRYCTL_DISABLE_SECMEM, 0);
+  gcry_control (GCRYCTL_INITIALIZATION_FINISHED, 0);
+  gcry_md_hd_t sha1_context;
+  gcry_md_open(&sha1_context, GCRY_MD_SHA1, GCRY_MD_FLAG_HMAC);
+  gcry_md_setkey(sha1_context, aws_access_key_secret, 
+                 strlen(aws_access_key_secret));
+  gcry_md_write(sha1_context, gp->c_str(), gp->length());  
+
+  unsigned char *digest= gcry_md_read(sha1_context, GCRY_MD_FLAG_HMAC);
+  char* cry= new char[(uint32_t) base64_needed_encoded_length(
+      SHA1_DIGEST_LENGTH) + 1];
+  (void) base64_encode(digest, SHA1_DIGEST_LENGTH, cry);
+
+  gcry_md_close(sha1_context);
+
+  returnstr->clear();
+  returnstr->append("AWS ");
+  returnstr->append(aws_access_key_id);
+  returnstr->append(":");
+  returnstr->append(cry);
+
+  delete[] cry;
+
+  return returnstr;
+}
+
+const std::string AWSS3Service::currentDateTime() const
+{
+  char st[40];
+  time_t t= time(NULL);
+  struct tm tm;
+  gmtime_r(&t, &tm);
+
+  /* TODO: set locale temporarily to "C" for call to strftime */
+  strftime(st, sizeof(st), "%a, %d %b %Y %T GMT", &tm);
+
+  return std::string(st);
+}
+
+long AWSS3Service::getBucket(std::string *result,
+                             int limit, const char *parameter)
+{
+  (void) limit;
+
+  std::string path= "/?";
+  char limit_no[10]= {0};
+  
+  snprintf(limit_no, sizeof(limit_no), "%d", limit);
+  path.append("max-keys=");
+  path.append(limit_no);
+
+  if (parameter)
+  {
+    path.append("&");
+    path.append(parameter);
+  }
+
+  HTTPHeaderMap httpRequestHeaders;
+
+  std::string t= currentDateTime();
+  httpRequestHeaders.insert(HTTPHeaderMap::value_type("Date", t));
+  connection->reset();
+  connection->setRequestMethod(HTTP::GET);
+  connection->setRequestURI(path);
+  connection->addRequestHeader("Date", t);
+  connection->setDebugOS(&std::cout);
+
+  std::string auth;
+  makeAWSS3Authorization(&auth, "GET", httpRequestHeaders, "");
+  connection->addRequestHeader("Authorization", auth);
+
+  bool res= connection->request();
+
+  if (!res)
+    return -1;
+
+  *result= connection->getResponse();
+
+  return connection->getResponseCode();
+}
+
+long AWSS3Service::headBucket(const char *parameter)
+{
+  std::string path= "/";
+
+  if (parameter)
+  {
+    path.append("?");
+    path.append(parameter);
+  }
+
+  HTTPHeaderMap httpRequestHeaders;
+
+  std::string t= currentDateTime();
+  httpRequestHeaders.insert(HTTPHeaderMap::value_type("Date", t));
+  connection->reset();
+  connection->setRequestMethod(HTTP::GET);
+  connection->setRequestURI(path);
+  connection->addRequestHeader("Date", t);
+
+  std::string auth;
+  makeAWSS3Authorization(&auth, "GET", httpRequestHeaders, "");
+  connection->addRequestHeader("Authorization", auth);
+
+  bool res= connection->request();
+ 
+  if (!res)
+    return -1;
+
+  return connection->getResponseCode();
+}
+
+long AWSS3Service::getItem(const char *key, std::string *result,
+                           const char *parameter)
+{
+  std::string path= "/";
+
+  if (key)
+  {
+    path.append(key);
+  }
+
+  if (parameter)
+  {
+    path.append("?");
+    path.append(parameter);
+  }
+
+  HTTPHeaderMap httpRequestHeaders;
+  connection->reset();
+  connection->setRequestMethod(HTTP::GET);
+  connection->setRequestURI(path);
+  httpRequestHeaders["Date"]= currentDateTime();
+  connection->addRequestHeader("Date", httpRequestHeaders["Date"]);
+
+  std::string auth;
+  makeAWSS3Authorization(&auth, "GET", httpRequestHeaders, key);
+  connection->addRequestHeader("Authorization", auth);
+
+  bool res= connection->request();
+
+  if (!res)
+    return -1;
+
+  result->clear();
+  result->append(connection->getResponse());
+
+  return connection->getResponseCode();
+}
+
+long AWSS3Service::headItem(const char *key, const char *parameter)
+{
+  std::string path= "/";
+
+  if (key)
+  {
+    path.append(key);
+  }
+
+  if (parameter)
+  {
+    path.append("?");
+    path.append(parameter);
+  }
+
+  HTTPHeaderMap httpRequestHeaders;
+  connection->reset();
+  connection->setRequestMethod(HTTP::GET);
+  connection->setRequestURI(path);
+  httpRequestHeaders["Date"]= currentDateTime();
+  connection->addRequestHeader("Date", httpRequestHeaders["Date"]);
+
+  std::string auth;
+  makeAWSS3Authorization(&auth, "GET", httpRequestHeaders, key);
+  connection->addRequestHeader("Authorization", auth);
+
+  bool res= connection->request();
+
+  if (!res)
+    return -1;
+
+  return connection->getResponseCode();
+}
+
+long AWSS3Service::putItem(const char *key, const char *content, size_t len,
+                           const char *parameter)
+{
+  std::string path= "/";
+
+  if (key)
+  {
+    path.append(key);
+  }
+
+  if (parameter)
+  {
+    path.append("?");
+    path.append(parameter);
+  }
+
+  HTTPHeaderMap httpRequestHeaders;
+  connection->reset();
+  connection->setRequestMethod(HTTP::PUT);
+  connection->setRequestURI(path);
+  httpRequestHeaders["Date"]= currentDateTime();
+  connection->addRequestHeader("Date", httpRequestHeaders["Date"]);
+
+  std::string auth;
+  makeAWSS3Authorization(&auth, "PUT", httpRequestHeaders, key);
+  connection->addRequestHeader("Authorization", auth);
+
+  bool res= connection->request(std::string(content, len));
+
+  if (!res)
+    return -1;
+
+  return connection->getResponseCode();
+}
+
+long AWSS3Service::deleteItem(const char *key, const char *parameter)
+{
+  std::string path= "/";
+
+  if (key)
+  {
+    path.append(key);
+  }
+
+  if (parameter)
+  {
+    path.append("?");
+    path.append(parameter);
+  }
+
+  HTTPHeaderMap httpRequestHeaders;
+  connection->reset();
+  connection->setRequestMethod(HTTP::DELETE);
+  connection->setRequestURI(path);
+  httpRequestHeaders["Date"]= currentDateTime();
+  connection->addRequestHeader("Date", httpRequestHeaders["Date"]);
+
+  std::string auth;
+  makeAWSS3Authorization(&auth, "DELETE", httpRequestHeaders, key);
+  connection->addRequestHeader("Authorization", auth);
+
+  bool res= connection->request();
+
+  if (!res)
+    return -1;
+
+  return connection->getResponseCode();
+}
+

=== added file 'plugin/cloud_storage/awss3_service.h'
--- plugin/cloud_storage/awss3_service.h	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/awss3_service.h	2010-08-19 16:21:41 +0000
@@ -0,0 +1,75 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef PLUGIN_CLOUD_AWSS3_SERVICE_H
+#define PLUGIN_CLOUD_AWSS3_SERVICE_H
+
+#include <libxml/xmlmemory.h>
+#include <libxml/parser.h>
+#include <libxml/tree.h>
+
+#include <string>
+#include <map>
+
+#include "cloud_storage_service.h"
+
+namespace HTTP {
+  class Connection;
+}
+
+class AWSS3Service: public CloudStorageService
+{
+public:
+  AWSS3Service(const std::string& bkt);
+  virtual ~AWSS3Service();
+
+  const char *serviceURL(void) const
+  {
+    return "s3.amazonaws.com";
+  }
+
+  virtual BucketInfo* parseBucketInfo(const std::string &content);
+  virtual long getBucket(std::string *result, 
+                         int maxKeys= 100, const char *parameter= NULL);
+  virtual long headBucket(const char *parameter= NULL);
+  virtual long getItem(const char *key, std::string *result,
+                       const char *parameter= NULL);
+  virtual long headItem(const char *key, const char *parameter= NULL);
+  virtual long putItem(const char *key, const char *content, size_t len,
+                       const char *parameter= NULL);
+  virtual long deleteItem(const char *key, const char *parameter= NULL);
+
+private:
+  typedef std::map<std::string, std::string> HTTPHeaderMap;
+
+  BucketInfo* parseBucketInfo(xmlDocPtr doc);
+  void parseItemKey(BucketInfo* bucket_info, xmlDocPtr doc, xmlNodePtr cur);
+
+  std::string* makeAWSS3CanonicalString(std::string *gp, const char *http_verb,
+                                        HTTPHeaderMap& headers, 
+                                        const char *aws_s3_objectname);
+
+  std::string* makeAWSS3Authorization(std::string *returnstr,
+                                      const char *http_verb, 
+                                      HTTPHeaderMap& headers,
+                                      const char *aws_s3_objectname);
+
+  const std::string currentDateTime() const;
+
+private:
+  HTTP::Connection* connection;
+};
+
+#endif /* undef PLUGIN_CLOUD_AWSS3_SERVICE_H */

=== added file 'plugin/cloud_storage/base64.cc'
--- plugin/cloud_storage/base64.cc	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/base64.cc	2010-08-19 16:21:41 +0000
@@ -0,0 +1,78 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "config.h"
+#include "base64.h"
+
+static char base64_table[]= "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+  "abcdefghijklmnopqrstuvwxyz"
+  "0123456789+/";
+
+int base64_needed_encoded_length(int length_of_data)
+{
+  int nb_base64_chars;
+  nb_base64_chars= (length_of_data + 2) / 3 * 4;
+
+  return (nb_base64_chars + /* base64 char incl padding */
+  (nb_base64_chars - 1) / 76 + /* newlines */
+  1); /* NUL termination of string */
+}
+
+int base64_encode(const void *src, size_t src_len, char *dst)
+{
+  const unsigned char *s= (const unsigned char*) src;
+  size_t i= 0;
+  size_t len= 0;
+
+  for (; i < src_len; len+= 4)
+    {
+      unsigned c= 0;
+
+      if (len == 76)
+        {
+          len= 0;
+          *dst++= '\n';
+        }
+
+      c= s[i++];
+      c <<= 8;
+
+      if (i < src_len)
+        c+= s[i];
+      c <<= 8;
+      i++;
+
+      if (i < src_len)
+        c+= s[i];
+      i++;
+
+      *dst++= base64_table[(c >> 18) & 0x3f];
+      *dst++= base64_table[(c >> 12) & 0x3f];
+
+      if (i > (src_len + 1))
+        *dst++= '=';
+      else
+        *dst++= base64_table[(c >> 6) & 0x3f];
+
+      if (i > src_len)
+        *dst++= '=';
+      else
+        *dst++= base64_table[(c >> 0) & 0x3f];
+    }
+  *dst= '\0';
+
+  return 0;
+}
+

=== added file 'plugin/cloud_storage/base64.h'
--- plugin/cloud_storage/base64.h	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/base64.h	2010-08-19 16:21:41 +0000
@@ -0,0 +1,32 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef PLUGIN_CLOUD_BASE64_H
+#define PLUGIN_CLOUD_BASE64_H
+
+#include <cstddef>
+
+extern int base64_needed_encoded_length(int length_of_data);
+
+/*
+Encode a data as base64.
+
+Note: We require that dst is pre-allocated to correct size.
+See base64_needed_encoded_length().
+*/
+extern int base64_encode(const void *src, size_t src_len, char *dst);
+
+#endif /* undef PLUGIN_CLOUD_BASE64_H */
+

=== added file 'plugin/cloud_storage/cloud_storage_service.h'
--- plugin/cloud_storage/cloud_storage_service.h	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/cloud_storage_service.h	2010-08-19 16:21:41 +0000
@@ -0,0 +1,84 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef PLUGIN_CLOUD_CLOUD_STORAGE_H
+#define PLUGIN_CLOUD_CLOUD_STORAGE_H
+
+#include <string>
+#include <vector>
+
+class BucketInfo
+{
+private:
+  BucketInfo(const BucketInfo&);
+  BucketInfo& operator=(const BucketInfo &);
+
+public:
+  BucketInfo() :
+    isTrucated(false), keys()
+  {
+  }
+
+  ~BucketInfo()
+  {
+  }
+
+  bool isTrucated;
+  std::vector<std::string> keys;
+};
+
+class CloudStorageService
+{
+public:
+  CloudStorageService(const std::string& bkt) : 
+    bucket(bkt)
+  {
+  }
+
+  virtual ~CloudStorageService()
+  {
+  }
+
+  void setKeyID(const std::string &key_id) {
+    keyID= key_id;
+  }
+
+  void setKeyValue(const std::string &key_value) {
+    keyValue= key_value;
+  }
+
+  virtual BucketInfo* parseBucketInfo(const std::string &content) = 0;
+  virtual long getBucket(std::string *result, 
+                         int maxKeys= 100, const char *parameter= NULL) = 0;
+  virtual long headBucket(const char *parameter= NULL) = 0;
+  virtual long getItem(const char *key, std::string *result,
+                       const char *parameter= NULL) = 0;
+  virtual long headItem(const char *key, const char *parameter= NULL) = 0;
+  virtual long putItem(const char *key, const char *content, size_t len,
+                       const char *parameter= NULL) = 0;
+  virtual long deleteItem(const char *key, const char *parameter= NULL) = 0;
+
+private:
+  CloudStorageService();
+  CloudStorageService& operator=(const CloudStorageService &);
+  CloudStorageService(const CloudStorageService&);
+
+protected:
+  std::string bucket;
+  std::string keyID;
+  std::string keyValue;
+};
+
+#endif /* undef PLUGIN_CLOUD_CLOUD_STORAGE_H */

=== added file 'plugin/cloud_storage/curl.cc'
--- plugin/cloud_storage/curl.cc	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/curl.cc	2010-08-19 16:21:41 +0000
@@ -0,0 +1,281 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "config.h"
+
+#include <cassert>
+#include <cstring>
+#include <string>
+
+#include <boost/algorithm/string.hpp>
+
+#include "curl.h"
+
+namespace HTTP
+{
+
+void ConnectionFactory::init()
+{
+  curl_global_init(CURL_GLOBAL_ALL);
+}
+
+void ConnectionFactory::reset()
+{
+  curl_global_cleanup();
+}
+
+Connection* ConnectionFactory::openHttpConnection(const std::string& host)
+{
+  return new Connection(host);
+}
+
+void ConnectionFactory::destroyHttpConnection(const Connection* http)
+{
+  delete http;
+}
+
+Connection::Connection(const std::string& h) :
+  curl(NULL), host(h), method(GET), requestHeaders(NULL), responseHeaders(),
+      responseCode(0), response(), debugOS(NULL)
+{
+  curl= curl_easy_init();
+  assert(curl);
+}
+
+Connection::~Connection()
+{
+  curl_easy_cleanup(curl);
+  curl= NULL;
+
+  curl_slist_free_all(requestHeaders);
+  requestHeaders= NULL;
+}
+
+void Connection::setRequestURI(const std::string& url)
+{
+  assert(!url.empty() && url[0] == '/');
+  curl_easy_setopt(curl, CURLOPT_URL, (host+url).c_str());
+}
+
+void Connection::setRequestMethod(METHOD m)
+{
+  this->method= m;
+}
+
+void Connection::addRequestHeader(const std::string& key,
+    const std::string& value)
+{
+  std::string header;
+  header.reserve(key.size() + value.size() + 1);
+  header.append(boost::trim_copy(key));
+  header.append(": ");
+  header.append(boost::trim_copy(value));
+
+  requestHeaders= curl_slist_append(requestHeaders, header.c_str());
+  assert(requestHeaders != NULL);
+}
+
+bool Connection::request(const std::string& data)
+{
+
+  assert(data.empty() || method == PUT);
+
+  curl_io_context io_ctx_rt;
+  curl_io_context io_ctx_st;
+
+  switch (method)
+  {
+    case GET:
+    {
+      io_ctx_rt.ioPos= 0;
+      io_ctx_rt.ioBuffer= &response;
+      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+          &Connection::curlWriteCallback);
+      curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &io_ctx_rt);
+      break;
+    }
+    case PUT:
+    {
+      io_ctx_rt.ioPos= 0;
+      io_ctx_rt.ioBuffer= &response;
+      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+          &Connection::curlWriteCallback);
+      curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &io_ctx_rt);
+
+      io_ctx_st.ioPos= 0;
+      io_ctx_st.ioBuffer= const_cast<std::string*> (&data);
+      curl_easy_setopt(curl, CURLOPT_READFUNCTION,
+          &Connection::curlReadCallback);
+      curl_easy_setopt(curl, CURLOPT_READDATA, (void *) &io_ctx_st);
+
+      curl_easy_setopt(curl, CURLOPT_PUT, true);
+      curl_easy_setopt(curl, CURLOPT_UPLOAD, true);
+      curl_easy_setopt(curl, CURLOPT_INFILESIZE, data.size());
+      break;
+    }
+    case HEAD:
+      curl_easy_setopt(curl, CURLOPT_NOBODY, true);
+      break;
+    case DELETE:
+      // be aware that 200, 202, and 204 are valid Ok response codes
+      curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
+      break;
+    default:
+      assert (0 && "Not implement http method");
+  }
+
+  curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
+  curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &responseHeaders);
+  curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, Connection::curlHeaderCallback);
+  //curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, Connection::curlDebugCallback);
+
+  errorBuffer[0]= '\0';
+  curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorBuffer);
+  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, requestHeaders);
+
+  CURLcode res= curl_easy_perform(curl);
+  if (res == CURLE_OK)
+  {
+    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);
+  }
+
+  return (res == CURLE_OK);
+}
+
+long Connection::getResponseCode() const
+{
+  return responseCode;
+}
+
+const std::string& Connection::getResponseHeader(const std::string& key) const
+{
+  HTTPHeaderIterator it= responseHeaders.find(key);
+  assert (it != responseHeaders.end());
+  return it->second;
+}
+
+Connection::HTTPHeaderIterator Connection::responseHeaderBegin() const
+{
+  return responseHeaders.begin();
+}
+
+Connection::HTTPHeaderIterator Connection::responseHeaderEnd() const
+{
+  return responseHeaders.end();
+}
+
+const std::string& Connection::getResponse() const
+{
+  assert(method != HEAD || method != PUT || response.empty());
+  return response;
+}
+
+void Connection::reset()
+{
+  curl_easy_reset(curl);
+
+  method= GET;
+
+  curl_slist_free_all(requestHeaders);
+  requestHeaders= NULL;
+
+  responseCode= 0;
+  response= "";
+  responseHeaders.clear();
+
+  debugOS= NULL;
+
+  errorBuffer[0]= '\0';
+}
+
+void Connection::setDebugOS(std::ostream* os)
+{
+  debugOS= os;
+  if (debugOS != NULL)
+  {
+    *debugOS << "******debug information for curl" << std::endl;
+    *debugOS << "CURL version: " << curl_version() << std::endl;
+  }
+}
+
+size_t Connection::curlHeaderCallback(void *ptr, size_t size, size_t nmemb,
+    void *stream)
+{
+  HTTPHeaderMap* headers= reinterpret_cast<HTTPHeaderMap*> (stream);
+  std::string response(reinterpret_cast<const char*> (ptr), size * nmemb);
+
+  boost::trim(response);
+
+  if (not response.empty() && response.find("HTTP/1.1") == std::string::npos
+      && response.find("HTTP/1.0") == std::string::npos)
+  {
+    std::string::size_type pos_of_colon= response.find(':');
+    assert(pos_of_colon != std::string::npos);
+    std::string key= response.substr(0, pos_of_colon);
+    std::string value= response.substr(pos_of_colon + 1);
+
+    boost::trim(key);
+    boost::trim(value);
+
+    headers->insert(HTTPHeaderMap::value_type(key, value));
+  }
+
+  return size * nmemb;
+}
+
+size_t Connection::curlWriteCallback(void *ptr, size_t size, size_t nmemb,
+    void *data)
+{
+  curl_io_context *ctx= (curl_io_context *) data;
+  size_t realsize= size * nmemb;
+
+  ctx->ioBuffer->append((const char *) ptr, realsize);
+  ctx->ioPos+= realsize;
+
+  return realsize;
+}
+
+size_t Connection::curlReadCallback(void *ptr, size_t size, size_t nmemb,
+    void *data)
+{
+  curl_io_context *ctx= (curl_io_context *) data;
+  size_t willsend= size * nmemb;
+
+  size_t maxtosend= ctx->ioBuffer->size() - ctx->ioPos;
+  if (willsend > maxtosend)
+    willsend= maxtosend;
+
+  memcpy(ptr, ctx->ioBuffer->c_str() + ctx->ioPos, willsend);
+  ctx->ioPos+= willsend;
+
+  return willsend;
+}
+
+int Connection::curlDebugCallback(CURL *, curl_infotype type, char *str,
+    size_t n, void *)
+{
+  (void)type;
+  (void)str;
+  (void)n;
+
+  return 0;
+}
+
+const char* Connection::getErrorMessage() const
+{
+  return errorBuffer;
+}
+
+} // end of namespace

=== added file 'plugin/cloud_storage/curl.h'
--- plugin/cloud_storage/curl.h	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/curl.h	2010-08-19 16:21:41 +0000
@@ -0,0 +1,130 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#ifndef PLUGIN_CLOUD_CURL_H
+#define PLUGIN_CLOUD_CURL_H
+
+#include <string>
+#include <map>
+
+#include <boost/utility.hpp>
+
+#include <curl/curl.h>
+
+namespace HTTP
+{
+
+enum METHOD
+{
+  GET= 0, HEAD, PUT, DELETE
+};
+
+class Connection;
+
+class ConnectionFactory
+{
+public:
+  /**
+   * NOTE
+   *  *) These two functions could be called in multiple threads.
+   *  *) These two functions should be called once in one application.
+   */
+  static void init();
+  static void reset();
+
+  static Connection* openHttpConnection(const std::string& host);
+  static void destroyHttpConnection(const Connection* http);
+};
+
+class Connection: boost::noncopyable
+{
+private:
+  typedef struct curl_io_context
+  {
+    std::string* ioBuffer;
+    size_t ioPos;
+  } curl_io_context;
+
+  typedef std::map<std::string, std::string> HTTPHeaderMap;
+
+public:
+  typedef HTTPHeaderMap::const_iterator HTTPHeaderIterator;
+  explicit Connection(const std::string& host);
+  ~Connection();
+
+  /**
+   * Specify requested url, must specify by user.
+   * You could re-use current connection to request multiple URIs.
+   */
+  void setRequestURI(const std::string& url);
+
+  /**
+   * Default GET method, if not set explicitly.
+   */
+  void setRequestMethod(METHOD method);
+
+  void addRequestHeader(const std::string& key, const std::string& value);
+
+  /**
+   * For form data, user should encoding the data.
+   * For GET/HEAD/DELETE method, ignore data.
+   */
+  bool request(const std::string& data= "");
+
+  /**
+   * @return zero if no server response code has been received.
+   */
+  long getResponseCode() const;
+
+  const std::string& getResponseHeader(const std::string& key) const;
+
+  HTTPHeaderIterator responseHeaderBegin() const;
+  HTTPHeaderIterator responseHeaderEnd() const;
+
+  const std::string& getResponse() const;
+
+  void reset();
+
+  /**
+   * set debug information to be output to @os.
+   */
+  void setDebugOS(std::ostream* os);
+
+  const char* getErrorMessage() const;
+
+private:
+  static size_t curlHeaderCallback(void *ptr, size_t size, size_t nmemb,
+                                   void *stream);
+  static int curlDebugCallback(CURL *curl, curl_infotype type, char *str,
+                               size_t n, void *vp);
+  static size_t curlWriteCallback(void *ptr, size_t size, size_t nmemb,
+                                  void *data);
+  static size_t curlReadCallback(void *ptr, size_t size, size_t nmemb,
+                                 void *data);
+private:
+  CURL *curl;
+  const std::string host;
+  METHOD method;
+  struct curl_slist *requestHeaders;
+  HTTPHeaderMap responseHeaders;
+  long responseCode;
+  std::string response;
+
+  std::ostream* debugOS;
+  char errorBuffer[CURL_ERROR_SIZE];
+};
+}
+
+#endif /* undef PLUGIN_CLOUD_CURL_H */

=== added file 'plugin/cloud_storage/ha_cloud.cc'
--- plugin/cloud_storage/ha_cloud.cc	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/ha_cloud.cc	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1187 @@
+/* Copyright (C) 2010 Wei Ye
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+#include "config.h"
+
+#include <drizzled/field.h>
+#include <drizzled/field/blob.h>
+#include <drizzled/field/timestamp.h>
+#include <drizzled/error.h>
+#include <drizzled/table.h>
+#include <drizzled/session.h>
+#include <drizzled/internal/my_sys.h>
+#include <drizzled/enum.h>
+#include <drizzled/sql_bitmap.h>
+
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <boost/algorithm/string.hpp>
+
+#include <fcntl.h>
+#include <ctype.h>
+#include <fstream>
+#include <cstring>
+
+#include "ha_cloud.h"
+
+#include "str_percent.h"
+#include "curl.h"
+#include "cloud_storage_service.h"
+#include "awss3_service.h"
+
+using namespace drizzled;
+using namespace std;
+
+#define CLOUD_EXT ".cld"
+
+/* Stuff for shares */
+static pthread_mutex_t cloud_mutex;
+
+#define BLOB_MEMROOT_ALLOC_SIZE 8192
+
+class CloudEngine;
+static CloudEngine *cloud_engine= NULL;
+static const char* ha_cloud_exts[]= { CLOUD_EXT, NULL };
+
+typedef drizzled::system_status_var SSV;
+
+static const char BUCKET_OPTION[]= "BUCKET";
+static const char KEY_ID_OPTION[]= "KEY_ID";
+static const char KEY_OPTION[]= "KEY_VALUE";
+static const char SERVICE_OPTION[]= "SERVICE";
+
+static const char AWSS3[]= "awss3";
+static const char RACKSPACE[]= "rackspace";
+
+CloudShare::CloudShare(const char *table_name_arg) :
+  table_name(table_name_arg), 
+  use_count(0), 
+  storageService(NULL)
+{
+  pthread_mutex_init(&mutex, MY_MUTEX_INIT_FAST);
+  thr_lock_init(&lock);
+}
+
+CloudShare::~CloudShare()
+{
+  delete storageService;
+  lock.deinit();
+  pthread_mutex_destroy(&mutex);
+}
+
+class CloudEngine: public drizzled::plugin::StorageEngine
+{
+  typedef std::map<std::string, CloudShare*> CloudMap;
+  CloudMap cloud_open_tables;
+
+public:
+  CloudEngine(const string &name_arg) :
+    drizzled::plugin::StorageEngine(name_arg,
+                                    drizzled::HTON_ALTER_NOT_SUPPORTED | 
+                                    drizzled::HTON_REQUIRE_PRIMARY_KEY | 
+                                    drizzled::HTON_SKIP_STORE_LOCK |
+                                    drizzled::HTON_AUTO_PART_KEY),
+                                    cloud_open_tables()
+  {
+    table_definition_ext= CLOUD_EXT;
+  }
+
+  virtual ~CloudEngine()
+  {
+    pthread_mutex_destroy(&cloud_mutex);
+    HTTP::ConnectionFactory::reset();
+  }
+
+  virtual drizzled::Cursor *create(drizzled::TableShare &table)
+  {
+    return new ha_cloud(*this, table);
+  }
+
+  int doCreateTable(drizzled::Session &session,
+                    drizzled::Table &table_arg,
+                    const drizzled::TableIdentifier &identifier, 
+                    drizzled::message::Table &proto);
+
+  int doGetTableDefinition(drizzled::Session &session,
+                           const drizzled::TableIdentifier &identifier,
+                           drizzled::message::Table &table_message);
+
+  void doGetTableNames(drizzled::CachedDirectory &directory,
+                       const SchemaIdentifier &,
+                       set<string> &set_of_names);
+
+  int doDropTable(drizzled::Session&, const drizzled::TableIdentifier &identifier);
+
+  int doRenameTable(drizzled::Session&, const drizzled::TableIdentifier&,
+                    const drizzled::TableIdentifier&);
+
+  void doGetTableIdentifiers(drizzled::CachedDirectory&,
+                             const drizzled::SchemaIdentifier&, 
+                             drizzled::TableIdentifiers&);
+
+  bool doDoesTableExist(drizzled::Session&,
+                        const drizzled::TableIdentifier &identifier);
+  
+  bool validateCreateTableOption(const std::string &key, const std::string &state);
+
+  const char** bas_ext() const
+  {
+    return ha_cloud_exts;
+  }
+
+  CloudShare *findOpenTable(const std::string &table_name);
+  bool addOpenTable(const std::string &table_name, CloudShare *);
+  void deleteOpenTable(const std::string &table_name);
+
+  uint32_t max_supported_keys() const  { return 1; }
+  uint32_t max_supported_key_parts() const { return 1; }
+
+  uint32_t max_supported_record_length() const
+  {
+    return HA_MAX_REC_LENGTH;
+  }
+
+  uint32_t max_supported_key_part_length() const 
+  { 
+    return MAX_PRIMARY_KEY_LENGTH; 
+  }
+
+  uint32_t index_flags(enum drizzled::ha_key_alg) const
+  {
+    return (HA_ONLY_WHOLE_INDEX | HA_KEYREAD_ONLY);
+  }
+};
+
+void CloudEngine::doGetTableNames(drizzled::CachedDirectory &directory,
+                                  const SchemaIdentifier &,
+                                  set<string> &set_of_names)
+{
+  drizzled::CachedDirectory::Entries entries= directory.getEntries();
+
+  for (drizzled::CachedDirectory::Entries::iterator entry_iter= entries.begin();
+       entry_iter != entries.end(); ++entry_iter)
+  {
+    drizzled::CachedDirectory::Entry *entry= *entry_iter;
+    const string *filename= &entry->filename;
+
+    assert(filename->size());
+
+    const char *ext= strchr(filename->c_str(), '.');
+
+    if (ext == NULL || my_strcasecmp(system_charset_info, ext, CLOUD_EXT) ||
+       (filename->compare(0, strlen(TMP_FILE_PREFIX), TMP_FILE_PREFIX) == 0))
+    {  }
+    else
+    {
+      char uname[NAME_LEN + 1];
+      uint32_t file_name_len;
+
+      file_name_len= TableIdentifier::filename_to_tablename(filename->c_str(), uname, sizeof(uname));
+      // TODO: Remove need for memory copy here
+      uname[file_name_len - sizeof(CLOUD_EXT) + 1]= '\0'; // Subtract ending, place NULL
+      set_of_names.insert(uname);
+    }
+  }
+}
+
+bool CloudEngine::doDoesTableExist(drizzled::Session&,
+                                   const drizzled::TableIdentifier &identifier)
+{
+  string proto_path(identifier.getPath());
+  proto_path.append(CLOUD_EXT);
+
+  if (access(proto_path.c_str(), F_OK))
+  {
+    return false;
+  }
+
+  return true;
+}
+
+bool CloudEngine::validateCreateTableOption(const std::string &key,
+                                            const std::string &state)
+{
+  // if buckety is not set, table name is used as bucket name
+  // if key or key_value is not set, will do anonymous access to cloud service
+  if (boost::iequals(key, SERVICE_OPTION)) 
+  {
+    if (boost::iequals(state, AWSS3))
+      return true;
+    else
+      return false;
+  }
+
+  return true;
+}
+
+int CloudEngine::doRenameTable(Session&, const TableIdentifier &from, const TableIdentifier &to)
+{
+  int error= 0;
+
+  for (const char **ext= bas_ext(); *ext ; ext++)
+  {
+    if (rename_file_ext(from.getPath().c_str(), to.getPath().c_str(), *ext))
+    {
+      if ((error=errno) != ENOENT)
+        break;
+      error= 0;
+    }
+  }
+
+  return error;
+}
+
+void CloudEngine::doGetTableIdentifiers(drizzled::CachedDirectory &directory,
+                                        const drizzled::SchemaIdentifier &schema_identifier,
+                                        drizzled::TableIdentifiers &set_of_identifiers)
+{
+  drizzled::CachedDirectory::Entries entries= directory.getEntries();
+
+  for (drizzled::CachedDirectory::Entries::iterator entry_iter= entries.begin(); entry_iter
+      != entries.end(); ++entry_iter)
+  {
+    drizzled::CachedDirectory::Entry *entry= *entry_iter;
+    const string *filename= &entry->filename;
+
+    assert(filename->size());
+
+    const char *ext= strchr(filename->c_str(), '.');
+
+    if (ext == NULL || my_strcasecmp(system_charset_info, ext, CLOUD_EXT)
+        || (filename->compare(0, strlen(TMP_FILE_PREFIX), TMP_FILE_PREFIX) == 0))
+    {
+    }
+    else
+    {
+      char uname[NAME_LEN + 1];
+      uint32_t file_name_len;
+
+      file_name_len= TableIdentifier::filename_to_tablename(filename->c_str(), uname,
+          sizeof(uname));
+      // TODO: Remove need for memory copy here
+      uname[file_name_len - sizeof(CLOUD_EXT) + 1]= '\0'; // Subtract ending, place NULL
+
+      set_of_identifiers.push_back(TableIdentifier(schema_identifier, uname));
+    }
+  }
+}
+
+int CloudEngine::doDropTable(drizzled::Session&,
+                             const drizzled::TableIdentifier &identifier)
+{
+  string new_path(identifier.getPath());
+
+  new_path+= CLOUD_EXT;
+
+  int error= unlink(new_path.c_str());
+
+  if (error != 0)
+  {
+    error= errno;
+  }
+
+  return error;
+}
+
+CloudShare *CloudEngine::findOpenTable(const std::string &table_name)
+{
+  CloudMap::iterator find_iter= cloud_open_tables.find(table_name);
+
+  if (find_iter != cloud_open_tables.end())
+    return (*find_iter).second;
+  else
+    return NULL;
+}
+
+bool CloudEngine::addOpenTable(const std::string &table_name, CloudShare *share)
+{
+  CloudMap::iterator find_iter= cloud_open_tables.find(table_name);
+
+  if (find_iter != cloud_open_tables.end())
+    return true;
+  else
+  {
+    cloud_open_tables[table_name]= share;
+    return false;
+  }
+}
+
+void CloudEngine::deleteOpenTable(const std::string &table_name)
+{
+  cloud_open_tables.erase(table_name);
+}
+
+int CloudEngine::doGetTableDefinition(drizzled::Session&,
+                                      const drizzled::TableIdentifier &identifier,
+                                      drizzled::message::Table &table_proto)
+{
+  string new_path;
+
+  new_path= identifier.getPath();
+  new_path+= CLOUD_EXT;
+
+  int fd= open(new_path.c_str(), O_RDONLY);
+
+  if (fd == -1)
+  {
+    return errno;
+  }
+
+  google::protobuf::io::ZeroCopyInputStream* input=
+      new google::protobuf::io::FileInputStream(fd);
+
+  if (not input)
+    return HA_ERR_CRASHED_ON_USAGE;
+
+  if (not table_proto.ParseFromZeroCopyStream(input))
+  {
+    close(fd);
+    delete input;
+    if (not table_proto.IsInitialized())
+    {
+      my_error(ER_CORRUPT_TABLE_DEFINITION, MYF(0),
+          table_proto.InitializationErrorString().c_str());
+      return ER_CORRUPT_TABLE_DEFINITION;
+    }
+
+    return HA_ERR_CRASHED_ON_USAGE;
+  }
+
+  delete input;
+  close(fd);
+
+  return EEXIST;
+}
+
+int CloudEngine::doCreateTable(drizzled::Session&,
+                               drizzled::Table &table_arg,
+                               const drizzled::TableIdentifier &identifier,
+                               drizzled::message::Table &proto)
+{
+  const drizzled::TableShare::Fields fields(table_arg.getShare()->getFields());
+  for (drizzled::TableShare::Fields::const_iterator iter= fields.begin();
+       iter != fields.end();
+       iter++)
+  {
+    if (not *iter) // Historical legacy for NULL array end.
+      continue;
+  }
+
+  string serialized_proto;
+  string new_path;
+
+  new_path= identifier.getPath();
+  new_path+= CLOUD_EXT;
+  fstream output(new_path.c_str(), ios::out | ios::binary);
+
+  if (!output)
+    return 1;
+
+  if (!proto.SerializeToOstream(&output))
+  {
+    output.close();
+    unlink(new_path.c_str());
+    return 1;
+  }
+
+  return 0;
+}
+
+ha_cloud::ha_cloud(drizzled::plugin::StorageEngine &engine_arg,
+                   drizzled::TableShare &table_arg) :
+  Cursor(engine_arg, table_arg)
+{
+  rnd_state_is_have= false;
+  key_built_buffer[0]= '\0';
+  current_key= key_built_buffer;
+  current_key_length= 0;
+
+  key_quoted[0]= '\0';
+}
+
+ha_cloud::~ha_cloud()
+{
+}
+
+CloudShare* ha_cloud::getShare(const char *table_name, drizzled::Table*)
+{
+  CloudShare* shared;
+
+  pthread_mutex_lock(&cloud_mutex);
+
+  if ((shared= cloud_engine->findOpenTable(table_name)) == NULL)
+  {
+    shared= new CloudShare(table_name);
+    if (shared == NULL)
+    {
+      pthread_mutex_unlock(&cloud_mutex);
+      return NULL;
+    }
+
+    shared->use_count= 0;
+    shared->table_name= table_name;
+
+    if (parse_connect_string(shared))
+    {
+      pthread_mutex_unlock(&cloud_mutex);
+      return NULL;
+    }
+
+    if (cloud_engine->addOpenTable(table_name, shared))
+    {
+      pthread_mutex_destroy(&shared->mutex);
+      delete shared;
+    }
+    thr_lock_init(&shared->lock);
+    pthread_mutex_init(&shared->mutex, MY_MUTEX_INIT_FAST);
+  }
+
+  assert(shared != NULL);
+  shared->use_count++;
+  pthread_mutex_unlock(&cloud_mutex);
+  return shared;
+}
+
+int ha_cloud::freeShare(CloudShare *shared)
+{
+  pthread_mutex_lock(&cloud_mutex);
+  if (!--shared->use_count)
+  {
+    cloud_engine->deleteOpenTable(shared->table_name);
+    shared->lock.deinit();
+    pthread_mutex_destroy(&shared->mutex);
+
+    delete shared;
+  }
+  pthread_mutex_unlock(&cloud_mutex);
+
+  return 0;
+}
+
+int ha_cloud::open(const char *name, int mode, uint test_if_locked)
+{
+  (void) mode;
+  (void) test_if_locked;
+
+  if (!(share= getShare(name, table)))
+    return (1);
+
+  lock.init(&share->lock);
+
+  // size of a length plus max size of a key
+  ref_length= sizeof(size_t) + MAX_CLOUD_SERVICE_KEY_LENGTH;
+
+  in_table_scan= false;
+  onlyPrimaryKeyFiledNeeded= false;
+  http_response_code= 0;
+
+  return (0);
+}
+
+int ha_cloud::close(void)
+{
+  return (freeShare(share));
+}
+
+int ha_cloud::parse_connect_string(CloudShare *shared)
+{
+  assert(table && shared);
+
+  string service;
+  string bucket;
+  string keyID;
+  string keyValue;
+  size_t num_engine_options= table->s->getTableProto()->engine().options_size();
+
+  for (size_t y= 0; y < num_engine_options; ++y)
+  {
+    string name= table->s->getTableProto()->engine().options(y).name();
+    string value= table->s->getTableProto()->engine().options(y).state();
+    
+    if (boost::iequals(name, BUCKET_OPTION))
+      bucket= value;
+    else if (boost::iequals(name, KEY_ID_OPTION))
+      keyID= value;
+    else if (boost::iequals(name, KEY_OPTION))
+      keyValue= value;
+    else if (boost::iequals(name, SERVICE_OPTION))
+      service= value;
+  }
+
+  // use table name as default bucket name
+  if (bucket.empty())
+    bucket= string(table->s->getTableName(),table->s->getTableNameSize());
+
+  if (boost::iequals(service, AWSS3))
+    shared->storageService= new AWSS3Service(bucket);
+
+  if (shared->storageService == NULL)
+    return (1);
+
+  shared->storageService->setKeyID(keyID);
+  shared->storageService->setKeyValue(keyValue);
+
+  return (0);
+}
+
+/*
+  Encode a buffer into the quoted format.
+*/
+
+int ha_cloud::encode_quote(unsigned char *)
+{
+  String attribute;
+
+  buffer.clear();
+  buffer.push_back('{');
+
+  Field *key_field= getPrimaryKeyField();
+
+  for (Field **field= table->getFields() ; *field ; field++)
+  {
+    const char *ptr;
+    const char *end_ptr;
+
+    if (*field == key_field)
+      continue;
+
+    buffer.append((*field)->field_name);
+    buffer.push_back(':');
+
+    if ((*field)->is_null())
+    {
+      (*field)->set_null();
+      buffer.append("null");
+      buffer.push_back(',');
+      continue;
+    }
+
+    (*field)->setReadSet();
+    (*field)->val_str(&attribute,&attribute);
+
+    if ((*field)->str_needs_quotes())
+    {
+      ptr= attribute.ptr();
+      end_ptr= attribute.length() + ptr;
+
+      buffer.push_back('"');
+
+      while (ptr < end_ptr)
+      {
+        if (*ptr == '"')
+        {
+          buffer.push_back('\\');
+          buffer.push_back('"');
+          *ptr++;
+        }
+        else if (*ptr == '\r')
+        {
+          buffer.push_back('\\');
+          buffer.push_back('r');
+          *ptr++;
+        }
+        else if (*ptr == '\\')
+        {
+          buffer.push_back('\\');
+          buffer.push_back('\\');
+          *ptr++;
+        }
+        else if (*ptr == '\n')
+        {
+          buffer.push_back('\\');
+          buffer.push_back('n');
+          *ptr++;
+        }
+        else
+          buffer.push_back(*ptr++);
+      }
+      buffer.push_back('"');
+    }
+    else
+    {
+      buffer.append(string(attribute.ptr(), attribute.length()));
+    }
+
+    buffer.push_back(',');
+  }
+  // Remove the comma
+  if (!buffer.empty())
+    buffer.erase(buffer.length() - 1);
+
+  buffer.push_back('}');
+
+  return (buffer.length());
+}
+
+/*
+  fill a row based on a csv string.
+*/
+int ha_cloud::fill_row(const std::string &row, unsigned char *buf)
+{
+  const char *end_offset= row.c_str() + row.length();
+  const char *curr_offset= row.c_str();
+  int error;
+  bool quoted= false;
+  Field *key_field= getPrimaryKeyField();
+  Field **field=table->getFields();
+
+  blobroot.free_root(MYF(drizzled::memory::MARK_BLOCKS_FREE));
+
+  // invalid json string
+  if (*curr_offset != '{' || *(end_offset - 1) != '}')
+    goto err;
+
+  curr_offset++; // skip '{'
+  end_offset--; // ignore '}'
+  memset(buf, 0, table->getShare()->null_bytes);
+
+  for (; *field; field++)
+  {
+    char curr_char;
+
+    if (*field == key_field)
+      continue;
+
+    quoted= false;
+    buffer.clear();
+    if (curr_offset >= end_offset)
+      goto err;
+    // skip filed name
+    size_t field_name_len= strlen((*field)->field_name);
+
+
+    if (strncmp((*field)->field_name, curr_offset, field_name_len) != 0)
+      goto err;
+    curr_offset+= field_name_len;
+    curr_char= *curr_offset;
+    // skip ':'
+    if (curr_char != ':')
+      goto err;
+    curr_char= *(++curr_offset);
+
+    if (curr_char == '"')
+    {
+      quoted= true;
+      curr_offset++; // Incrementpast the first quote
+
+      for(; curr_offset < end_offset; curr_offset++)
+      {
+        curr_char= *curr_offset;
+        if (curr_char == '"' &&
+            (curr_offset == end_offset - 1 ||
+             *(curr_offset + 1) == ','))
+        {
+          curr_offset+= 2; // Move past the , and "
+          break;
+        }
+        if (curr_char == '\\' && curr_offset != (end_offset - 1))
+        {
+          curr_offset++;
+          curr_char= *curr_offset;
+          if (curr_char == 'r')
+            buffer.push_back('\r');
+          else if (curr_char == 'n' )
+            buffer.push_back('\n');
+          else if (curr_char == '\\' || curr_char == '"')
+            buffer.push_back(curr_char);
+          else  /* This could only happed with an externally created file */
+          {
+            buffer.push_back('\\');
+            buffer.push_back(curr_char);
+          }
+        }
+        else // ordinary symbol
+        {
+         if (curr_offset == end_offset - 1)
+            goto err; // should end with '"'
+          buffer.push_back(curr_char);
+        }
+      }
+    }
+    else
+    {
+      for(; curr_offset < end_offset; curr_offset++)
+      {
+        curr_char= *curr_offset;
+        if (curr_char == ',')
+        {
+          curr_offset++;       // Skip the ,
+          break;
+        }
+        buffer.push_back(curr_char);
+      }
+    }
+
+    // the field is null
+    if (!quoted && strncmp(buffer.c_str(), "null", 4) == 0)
+    {
+      (*field)->set_null();
+      continue;
+    }
+
+    (*field)->setWriteSet();
+    if ((*field)->store(buffer.c_str(), buffer.length(), &drizzled::my_charset_bin))
+      goto err;
+
+    if ((*field)->flags & BLOB_FLAG)
+    {
+      Field_blob *blob= *(Field_blob**) field;
+      unsigned char *src, *tgt;
+      uint32_t length, packlength;
+
+      packlength= blob->pack_length_no_ptr();
+      length= blob->get_length(blob->ptr);
+      memcpy(&src, blob->ptr + packlength, sizeof(char*));
+      if (src)
+      {
+        tgt= (unsigned char*) blobroot.alloc_root(length);
+        memmove(tgt, src, length);
+        memcpy(blob->ptr + packlength, &tgt, sizeof(char*));
+      }
+    }
+  }
+  error= 0;
+
+err:
+  for (; *field; field++)
+  {
+    if ((*field)->is_null())
+      (*field)->set_null();
+    else
+      (*field)->set_default();
+  }
+
+  return(error);
+}
+
+int ha_cloud::drizzle_error_code()
+{
+  int rc= 0;
+
+  if (http_response_code == 0)
+    rc= 0;
+  else if (http_response_code == -1)
+    rc= drizzled::ER_CONNECT_TO_FOREIGN_DATA_SOURCE;
+  else if (http_response_code == 401)
+    rc= drizzled::ER_PASSWORD_ANONYMOUS_USER;
+  else if (http_response_code == 403)
+    rc= drizzled::ER_PASSWORD_NOT_ALLOWED;
+  else if (http_response_code == 404)
+    rc= HA_ERR_END_OF_FILE;
+  else if ((http_response_code < 200) || (http_response_code > 299))
+    rc= drizzled::ER_UNKNOWN_ERROR;
+
+  return(rc);
+}
+
+inline drizzled::Field* ha_cloud::getPrimaryKeyField()
+{
+  return table->key_info[table->s->getPrimaryKey()].key_part->field;
+}
+
+int ha_cloud::doInsertRecord(unsigned char* buf)
+{
+  uint rc= 0;
+
+  drizzled::Field *key_field= getPrimaryKeyField();
+  make_key(key_field);
+
+  encode_quote(buf);
+  http_response_code= share->storageService->putItem(key_quoted, 
+     buffer.c_str(), buffer.size());
+  rc= drizzle_error_code();
+
+  return(rc);
+}
+
+int ha_cloud::doUpdateRecord(const unsigned char*, unsigned char *new_data)
+{
+  int rc= 0;
+
+  encode_quote(new_data);
+
+  // We need to save our old key to make sure that the primary key has
+  // not been updated if it has we will need to delete first.
+  char prev_key[1024 + 1];
+  memcpy(prev_key, current_key, current_key_length);
+  size_t prev_key_length= current_key_length;
+
+  // now grab the new key out of table, and stuff it into current_key
+  drizzled::Field *key_field= getPrimaryKeyField();
+  make_key(key_field);
+
+  if (current_key_length != prev_key_length || memcmp(current_key, prev_key,
+      current_key_length))
+  {
+    if (onlyPrimaryKeyFiledNeeded)
+    {
+      // if onlyPrimaryKeyFiledNeeded, row buffer will be empty.
+      // we need first fetch the itme content, since s3 
+      // do not provide a method to update item key
+      std::string item_content;
+
+      str_percent(key_quoted, sizeof(key_quoted),
+          (const unsigned char *) prev_key, prev_key_length, "/"); 
+      http_response_code= share->storageService->getItem(key_quoted, &item_content);
+      rc= drizzle_error_code();
+
+      if (rc != 0)
+        return rc;
+      
+      // now set item for the new key as the old item just fetched
+      str_percent(key_quoted, sizeof(key_quoted),
+          (const unsigned char *) current_key, current_key_length, "/");
+      http_response_code= share->storageService->putItem(key_quoted, 
+          item_content.c_str(),item_content.size());
+    }
+    else
+      http_response_code= share->storageService->putItem(key_quoted, 
+         buffer.c_str(), buffer.size());
+
+    rc= drizzle_error_code();
+    
+    if (rc != 0)
+      return rc;
+
+    // delete based on prev_key, prev_key_length
+    str_percent(key_quoted, sizeof(key_quoted),
+        (const unsigned char *) prev_key, prev_key_length, "/");
+    http_response_code= share->storageService->deleteItem(key_quoted);
+    rc= drizzle_error_code();
+  }
+  else
+  {
+    http_response_code= share->storageService->putItem(key_quoted,  
+        buffer.c_str(), buffer.size());
+    rc= drizzle_error_code();
+  }
+
+  return (rc);
+}
+
+int ha_cloud::doDeleteRecord(const unsigned char*)
+{
+  uint rc= 0;
+
+  drizzled::Field *key_field= getPrimaryKeyField();
+  make_key(key_field);
+
+  http_response_code= share->storageService->deleteItem(key_quoted);
+  rc= drizzle_error_code();
+
+  return (rc);
+}
+
+int ha_cloud::doStartIndexScan(uint idx, bool sorted) 
+{
+  (void)sorted;
+  active_index= idx;
+
+  onlyPrimaryKeyFiledNeeded= true;
+  blobroot.init_alloc_root(BLOB_MEMROOT_ALLOC_SIZE);
+
+  drizzled::Field *key_field= getPrimaryKeyField();
+  for (Field **field=table->getFields(); *field; field++)
+  {
+    if (*field != key_field && ((*field)->isReadSet() || (*field)->isWriteSet()))
+    {
+      onlyPrimaryKeyFiledNeeded= false;
+      break;
+    }
+  }
+
+  return 0;
+}
+
+int ha_cloud::doEndIndexScan()
+{
+  onlyPrimaryKeyFiledNeeded= false;
+  blobroot.free_root(MYF(0));
+
+  return 0;
+}
+
+int ha_cloud::index_read(unsigned char* buf, const unsigned char* key,
+    uint key_len, drizzled::ha_rkey_function)
+{
+  uint rc= 0;
+
+  drizzled::Field *key_field= getPrimaryKeyField();
+  key_field->set_key_image(key, key_len);
+  key_field->set_notnull();
+
+  find_row(buf, key_field);
+  rc= drizzle_error_code();
+
+  return (rc);
+}
+
+void ha_cloud::rnd_state_dealloc(void)
+{
+  rnd_state_is_have= false;
+  rnd_state_is_trunc= false;
+  rnd_state_itemkeys.clear();
+  rnd_state_ndx= 0;
+}
+
+void ha_cloud::update_rnd_state(BucketInfo *itemList)
+{
+  if (itemList)
+  {
+    rnd_state_is_trunc= itemList->isTrucated;
+    rnd_state_itemkeys= itemList->keys; 
+    rnd_state_is_have= true;
+  }
+}
+
+int ha_cloud::bucket_look(void)
+{
+  uint rc= 0;
+  string bucket_info;
+
+  // reset rnd_state_ndx for each bucket_look
+  rnd_state_ndx= 0; 
+
+  if (rnd_state_is_have)
+  {
+    if (rnd_state_is_trunc)
+    {
+      // there is more to read
+      rnd_state_is_trunc= false;
+      char key_marker[MAX_CLOUD_SERVICE_KEY_LENGTH + 1]= {0};
+
+      str_percent(key_marker, sizeof(key_marker),
+          (const unsigned char *) rnd_state_itemkeys.back().c_str(), 
+          rnd_state_itemkeys.back().length(), "/");
+
+      string marker= string("marker=") + string(key_marker);
+      http_response_code= share->storageService->getBucket(&bucket_info, 2, marker.c_str());
+
+      rc= drizzle_error_code();
+      if (rc)
+        return rc;
+
+      BucketInfo *itemList= share->storageService->parseBucketInfo(bucket_info);
+      if (itemList == NULL)
+        return (1);
+
+      update_rnd_state(itemList);
+      delete itemList;
+    } 
+    else // no more keys
+      rnd_state_dealloc();
+  }
+  else
+  {
+    // we're starting a bucket key scan
+    http_response_code= share->storageService->getBucket(&bucket_info);
+
+    if (http_response_code != 200)
+      return drizzle_error_code();
+
+    BucketInfo *itemList= share->storageService->parseBucketInfo(bucket_info);
+    if (itemList == NULL)
+      return (1);
+
+    update_rnd_state(itemList);
+    delete itemList;
+  }
+
+  return (0);
+}
+
+int ha_cloud::doStartTableScan(bool)
+{
+  int rc= 0;
+
+  if (in_table_scan)
+    doEndTableScan();
+
+  in_table_scan= true;
+  onlyPrimaryKeyFiledNeeded= true;
+  blobroot.init_alloc_root(BLOB_MEMROOT_ALLOC_SIZE);
+
+  drizzled::Field *key_field= getPrimaryKeyField();
+  for (Field **field=table->getFields(); *field; field++)
+  {
+    if (*field != key_field && ((*field)->isReadSet() || (*field)->isWriteSet()))
+    {
+      onlyPrimaryKeyFiledNeeded= false;
+      break;
+    }
+  }
+
+  rc= bucket_look();
+  
+  if (rc)
+    rnd_state_dealloc();
+
+  return (rc);
+}
+
+int ha_cloud::doEndTableScan()
+{
+  blobroot.free_root(MYF(0));
+  rnd_state_dealloc();
+  in_table_scan= false;
+  onlyPrimaryKeyFiledNeeded= false;
+
+  return (0);
+}
+
+int ha_cloud::rnd_next(unsigned char* buf)
+{
+  while (true) 
+  {
+    if (rnd_state_ndx == (int)rnd_state_itemkeys.size())
+    {
+      if (!rnd_state_is_trunc)
+        return (HA_ERR_END_OF_FILE);
+
+      // keys are truncated, bucket_look() again
+      if (bucket_look())
+        return (HA_ERR_END_OF_FILE);
+      
+      // no keys anymore
+      if (rnd_state_itemkeys.empty())
+        return (HA_ERR_END_OF_FILE);
+    }
+
+    drizzled::Field *key_field= getPrimaryKeyField();
+    key_field->setWriteSet();
+    key_field->store(rnd_state_itemkeys[rnd_state_ndx].c_str(),
+                     rnd_state_itemkeys[rnd_state_ndx].length(), 
+                     &drizzled::my_charset_bin);
+    key_field->set_notnull();
+
+    rnd_state_ndx++;
+    
+    if (!find_row(buf, key_field))
+      break;
+  }
+
+  return (0);
+}
+
+void ha_cloud::position(const unsigned char*)
+{
+  drizzled::Field *key_field= getPrimaryKeyField();
+  unsigned int key_length= key_field->data_length();
+
+  // ref and ref_length are inherited members of this class
+  memcpy(ref, &key_length, sizeof(unsigned int));
+  memcpy(ref + sizeof(unsigned int), key_field->ptr, key_length);
+}
+
+int ha_cloud::rnd_pos(unsigned char* buf, unsigned char* pos)
+{
+  drizzled::Field *key_field= getPrimaryKeyField();
+
+  uint32_t key_length;
+  memcpy(&key_length, pos, sizeof(unsigned int));
+  key_field->store((char*) pos + sizeof(unsigned int), key_length,
+      &drizzled::my_charset_bin);
+
+  find_row(buf, key_field);
+
+  return drizzle_error_code();
+}
+
+void ha_cloud::make_key(drizzled::Field *primary)
+{
+  char *end_ptr= key_built_buffer;
+
+  char attribute_buffer[1024];
+  drizzled::String attribute(attribute_buffer, sizeof(attribute_buffer),
+      &drizzled::my_charset_bin);
+
+  // key_build_buffer is a member of this class
+  // current_key is a member of this class
+  current_key= key_built_buffer;
+
+  primary->setReadSet();
+
+  // magically transform the primary key data into a string
+  // no really, magic, have brian explain this method to me one more time
+  primary->val_str(&attribute, &attribute);
+
+  // copy that string data into key_built_buffer (pointed to by end_ptr)
+  // key_built_buffer is a member of this class
+  memcpy(end_ptr, attribute.ptr(), attribute.length());
+  end_ptr+= attribute.length();
+
+  // current_key_length is a member of this class
+  current_key_length= (size_t) (end_ptr - key_built_buffer);
+
+  // make URL quoted version of the current key
+  str_percent(key_quoted, sizeof(key_quoted),
+      (const unsigned char *) current_key, current_key_length, "/");
+}
+
+int ha_cloud::find_row(unsigned char *buf, drizzled::Field *primary)
+{
+  int rc= 0;
+  make_key(primary);
+
+  // if only primary key is needed, no need to request cloud service
+  if (onlyPrimaryKeyFiledNeeded)
+  {
+    http_response_code= 0;
+    return rc;
+  }
+
+  string item_content;
+  http_response_code= share->storageService->getItem(key_quoted, &item_content);
+
+  if ((http_response_code >= 200) && (http_response_code < 300))
+    rc= fill_row(item_content, buf);
+  else
+    rc= drizzle_error_code();
+
+  return rc;
+}
+
+int ha_cloud::info(uint)
+{
+  if (stats.records < 2)
+    stats.records= 2;
+
+  return (0);
+}
+
+static int cloud_init_func(drizzled::module::Context &context)
+{
+  cloud_engine= new CloudEngine("CLOUD_STORAGE");
+  context.add(cloud_engine);
+
+  HTTP::ConnectionFactory::init();
+
+  pthread_mutex_init(&cloud_mutex, MY_MUTEX_INIT_FAST);
+  return 0;
+}
+
+DRIZZLE_DECLARE_PLUGIN {
+  DRIZZLE_VERSION_ID,
+  "CLOUD_STORAGE",
+  "1.0",
+  "Wei Ye",
+  "cloud storage engine",
+  drizzled::PLUGIN_LICENSE_GPL,
+  cloud_init_func, /* Plugin Init */
+  NULL, /* system variables */
+  NULL /* config options */
+}
+
+DRIZZLE_DECLARE_PLUGIN_END;

=== added file 'plugin/cloud_storage/ha_cloud.h'
--- plugin/cloud_storage/ha_cloud.h	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/ha_cloud.h	2010-08-19 16:21:41 +0000
@@ -0,0 +1,148 @@
+/* Copyright (C) 2010 Wei Ye
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
+
+
+#ifndef PLUGIN_CLOUD_HA_CLOUD_H
+#define PLUGIN_CLOUD_HA_CLOUD_H
+
+#include <drizzled/table_share.h>
+#include <drizzled/cursor.h>
+#include <drizzled/thr_lock.h>
+#include <drizzled/sql_string.h>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include <string>
+#include <vector>
+
+static const int CLOUD_VERSION= 1;
+static const int MAX_CLOUD_SERVICE_KEY_LENGTH= 1024;
+static const int MAX_PRIMARY_KEY_LENGTH= MAX_CLOUD_SERVICE_KEY_LENGTH * 4 / 3;
+
+class CloudStorageService;
+class BucketInfo;
+
+class CloudShare 
+{
+  CloudShare();
+  CloudShare(const CloudShare &);
+  CloudShare& operator=(const CloudShare &);
+public:
+  explicit CloudShare(const char *name);
+  ~CloudShare();
+  
+  std::string table_name;
+  uint32_t use_count;
+  CloudStorageService *storageService;
+  pthread_mutex_t mutex;
+  drizzled::THR_LOCK lock;
+};
+
+class ha_cloud: public drizzled::Cursor
+{
+private:
+  drizzled::THR_LOCK_DATA lock;  /* Drizzle lock */
+  CloudShare *share;  /* Shared lock info */
+
+  // an key can be up to 1024 bytes long
+  char key_built_buffer[MAX_CLOUD_SERVICE_KEY_LENGTH + 1];
+  char *current_key;
+  size_t current_key_length;
+  char key_quoted[MAX_CLOUD_SERVICE_KEY_LENGTH + 1];
+
+  // state for a table scan
+  bool rnd_state_is_have;
+  bool rnd_state_is_trunc;
+  int rnd_state_ndx;
+  std::vector<std::string> rnd_state_itemkeys;
+  bool in_table_scan;
+  bool onlyPrimaryKeyFiledNeeded;
+
+  long http_response_code;
+
+  drizzled::memory::Root blobroot;
+  std::string buffer;
+
+private:
+  void update_rnd_state(BucketInfo *bucket_info);
+  void rnd_state_dealloc(void);
+
+  int bucket_look(void);
+
+  int drizzle_error_code();
+  
+  inline drizzled::Field* getPrimaryKeyField();
+  void make_key(drizzled::Field *primary);
+
+  int parse_connect_string (CloudShare *shared);
+  int open_connection(void);
+
+  int encode_quote(unsigned char *);
+  int fill_row(const std::string &row, unsigned char *buf);
+  int find_row(unsigned char *buf, drizzled::Field *primary);
+
+public:
+  ha_cloud(drizzled::plugin::StorageEngine &engine, drizzled::TableShare &table_arg);
+  ~ha_cloud();
+
+  /** @brief
+    The name that will be used for display purposes.
+   */
+  const char *table_type() const { return "CLOUD"; }
+
+  /*
+     Called in test_quick_select to determine if indexes should be used.
+   */
+  virtual double scan_time() { return (double) (stats.records+stats.deleted) / 20.0+10; }
+
+  uint32_t index_flags(uint inx, uint part, bool all_parts) const
+  {
+    (void)inx;
+    (void)part;
+    (void)all_parts;
+	  
+    return HA_ONLY_WHOLE_INDEX;
+  }
+
+  const char *index_type(uint inx) { (void)inx; return ("HASH"); }
+  int doStartIndexScan(uint idx, bool sorted);
+  int index_read(unsigned char* buf, const unsigned char* key, uint key_len,
+                 drizzled::ha_rkey_function find_flag);
+  int doEndIndexScan();
+
+  int open(const char *name, int mode, uint test_if_locked);  
+  int close(void);
+  CloudShare* getShare(const char* table_name, drizzled::Table* table);
+  int freeShare( CloudShare* share);                    
+
+  int info(uint);  
+  int doStartTableScan(bool scan);                                     
+  int rnd_next(unsigned char *buf); 
+  void position(const unsigned char *record);                                      
+  int rnd_pos(unsigned char * buf, unsigned char *pos);                         
+  int doEndTableScan();
+
+  int doInsertRecord(unsigned char *buf);
+  int doDeleteRecord(const unsigned char *buf);
+  int doUpdateRecord(const unsigned char *old_data, unsigned char *new_data);
+
+  void get_auto_increment(uint64_t, uint64_t,
+                          uint64_t,
+                          uint64_t *,
+                          uint64_t *) {}
+};
+
+#endif /* PLUGIN_CLOUD_HA_CLOUD_H */

=== added file 'plugin/cloud_storage/plugin.ac'
--- plugin/cloud_storage/plugin.ac	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/plugin.ac	2010-08-19 16:21:41 +0000
@@ -0,0 +1,3 @@
+PANDORA_HAVE_LIBXML2
+PANDORA_HAVE_LIBCURL
+PANDORA_HAVE_LIBGCRYPT

=== added file 'plugin/cloud_storage/plugin.ini'
--- plugin/cloud_storage/plugin.ini	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/plugin.ini	2010-08-19 16:21:41 +0000
@@ -0,0 +1,8 @@
+[plugin]
+title=Cloud Storage Engine
+description=Cloud Storage Engine
+headers=ha_cloud.h str_percent.h curl.h cloud_storage_service.h awss3_service.h base64.h
+sources=ha_cloud.cc str_percent.cc curl.cc awss3_service.cc base64.cc
+load_by_default=yes
+build_conditional="x${ac_cv_libxml2}" = "xyes" -a "x${ac_cv_libcurl}" = "xyes"
+ldflags=${LTLIBXML2} ${LTLIBCURL} ${LTLIBGCRYPT}

=== added file 'plugin/cloud_storage/str_percent.cc'
--- plugin/cloud_storage/str_percent.cc	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/str_percent.cc	2010-08-19 16:21:41 +0000
@@ -0,0 +1,67 @@
+/* str_percent.cc -- create a C backslashed escaped string
+
+   Copyright (C) 2007 by Mark Atwood <[e-mail]>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */
+
+#include <string.h>
+#include "str_percent.h"
+
+char *str_percent (char *dst, size_t dstlen,
+		   const unsigned char *src, size_t srclen,
+		   const char *alsosafe)
+{
+  static char hexit[]= { '0', '1', '2', '3', '4', '5', '6', '7',
+			  '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
+  size_t i;  /* index down the dst */
+  size_t n;  /* index down the src */
+
+  // gcc generate Wlogical-op warns about strchr
+  // here we simulate strchr by a scan
+  size_t j;
+  size_t len= strlen(alsosafe);
+  const char *target;
+
+  for (i=0,n=0; n<srclen; n++) {
+
+	target= NULL;
+	if (src[n]) {
+      for (j= 0; j < len; j++) {
+        if (alsosafe[j] == src[n]) {
+          target= alsosafe + j;
+          break;
+        }
+      }
+	}
+
+    /* Worst case, need 4 dst bytes for the next src byte. 
+       percent hexit hexit null */
+    if ((dstlen - i) < 4) { dst[i]= '\0'; return dst; }
+
+    if (((src[n] >= 'A') && (src[n] <= 'Z'))
+	|| ((src[n] >= 'a') && (src[n] <= 'z'))
+	|| ((src[n] >= '0') && (src[n] <= '9'))) {
+      dst[i++]= src[n];
+    } else if ((src[n]) && (target != NULL)) {
+      dst[i++]= src[n];
+    } else {
+      dst[i++]= '%';
+      dst[i++]= hexit[(src[n] >> 4) & 0x0f];
+      dst[i++]= hexit[src[n] & 0x0f];
+    }
+    dst[i]= '\0';
+  }
+  return dst;
+}

=== added file 'plugin/cloud_storage/str_percent.h'
--- plugin/cloud_storage/str_percent.h	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/str_percent.h	2010-08-19 16:21:41 +0000
@@ -0,0 +1,36 @@
+/* str_percent.h -- create a C backslashed escaped string
+
+   Copyright (C) 2007 by Mark Atwood <[e-mail]>
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software Foundation,
+   Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.  */
+
+#ifndef _INCLUDE_STRPERCENT
+#define _INCLUDE_STRPERCENT
+
+/* Copy src into dst, converting it a C style backslashed escaped
+   string.  All control characters, high bit characters, the ASCII
+   NULL, and all whitespace except the space character itself will be
+   converted. The backslash and the quotation characters will be
+   escaped as well. If the dst buffer isn't long enough, as much as
+   will fit will be done.  The dst buffer will be null terminated,
+   so it can safely be treated as a C string.  The dst buffer will
+   also be returned.
+*/   
+
+extern char *str_percent (char *dst, size_t dstlen,
+		   const char unsigned *src, size_t srclen,
+		   const char *alsosafe);
+
+#endif /* ndef _INCLUDE_STRPERCENT */

=== added directory 'plugin/cloud_storage/tests'
=== added directory 'plugin/cloud_storage/tests/r'
=== added file 'plugin/cloud_storage/tests/r/basic_create_table.result'
--- plugin/cloud_storage/tests/r/basic_create_table.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/basic_create_table.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,19 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a varchar(20), b int not null)  ENGINE=cloud_storage;
+ERROR 42000: This table type requires a primary key
+CREATE TABLE t1 (a varchar(20), b int not null, constraint c primary key (a, b))  ENGINE=cloud_storage;
+ERROR 42000: Too many key parts specified; max 1 parts allowed
+CREATE TABLE t1 (a varchar(20) primary key, b int not null)  ENGINE=cloud_storage SERVICE=unknow;
+ERROR HY000: Unknown table engine option key/pair SERVICE = unknow.
+CREATE TABLE t1 (a varchar(342) primary key, b int not null)  ENGINE=cloud_storage;
+ERROR 42000: Specified key was too long; max key length is 1365 bytes
+CREATE TABLE t1 (a varchar(20) primary key, b int not null)  ENGINE=cloud_storage;
+CREATE TABLE t2 (a varchar(20) primary key,b int not null, c blob not null)  ENGINE=cloud_storage;
+CREATE TABLE t3 (a varchar(341) primary key)  ENGINE=cloud_storage;
+CREATE TABLE t4 (a int primary key)  ENGINE=cloud_storage;
+CREATE TABLE t5 (a int primary key)  ENGINE=cloud_storage SERVICE=awss3;
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;
+DROP TABLE t5;

=== added file 'plugin/cloud_storage/tests/r/basic_delete.result'
--- plugin/cloud_storage/tests/r/basic_delete.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/basic_delete.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,24 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+DELETE FROM t1;
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (444, "ddd");
+INSERT INTO t1 VALUES (555, "eee");
+SELECT * FROM t1;
+a	b
+222	bbb
+444	ddd
+555	eee
+DELETE FROM t1 WHERE a>=444;
+DELETE FROM t1 WHERE b="bbb";
+SELECT * FROM t1;
+a	b
+DELETE FROM t1 WHERE a!=111;
+SELECT * FROM t1;
+a	b
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/r/basic_insert.result'
--- plugin/cloud_storage/tests/r/basic_insert.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/basic_insert.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+DELETE FROM t1;
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (333, "ccc");
+SELECT * FROM t1;
+a	b
+222	bbb
+333	ccc
+DELETE FROM t1;
+INSERT INTO t1 VALUES (111, "aaa");
+SELECT * FROM t1;
+a	b
+111	aaa
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/r/basic_select.result'
--- plugin/cloud_storage/tests/r/basic_select.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/basic_select.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,18 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+DELETE FROM t1;
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (444, "ddd");
+INSERT INTO t1 VALUES (555, "eee");
+SELECT * FROM t1;
+a	b
+222	bbb
+444	ddd
+555	eee
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/r/basic_update.result'
--- plugin/cloud_storage/tests/r/basic_update.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/basic_update.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+DELETE FROM t1;
+INSERT INTO t1 VALUES (111, "aaa");
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (333, "ccc");
+UPDATE t1 SET a=444 WHERE a=222;
+UPDATE t1 SET a=555 WHERE b="bbb";
+UPDATE t1 SET b="ddd" WHERE a=111;
+SELECT * FROM t1;
+a	b
+111	ddd
+333	ccc
+555	bbb
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/r/multi_column.result'
--- plugin/cloud_storage/tests/r/multi_column.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/multi_column.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,26 @@
+drop table if exists t1;
+CREATE TABLE t1 (
+s3id varchar(20) not null primary key, 
+s3val1 blob not null, 
+s3val2 blob not null, 
+s3val3 int not null default 0) 
+engine=CLOUD_STORAGE 
+BUCKET="pku-cactus_cactus" 
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ" 
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV" 
+SERVICE="awss3";
+DELETE FROM t1;
+INSERT INTO t1 VALUES ("aaaa", "aaaa", "aaaa", 0);
+INSERT INTO t1 VALUES ("bbbb", "bbbb", "bbbb", 1);
+INSERT INTO t1 VALUES ("cccc", "cccc", "cccc", 0);
+UPDATE t1 SET s3val1 = "xxxx" WHERE s3id = "aaaa";
+DELETE FROM t1 WHERE s3id = "bbbb";
+SELECT * FROM t1 WHERE s3id = "cccc";
+s3id	s3val1	s3val2	s3val3
+cccc	cccc	cccc	0
+SELECT * FROM t1;
+s3id	s3val1	s3val2	s3val3
+aaaa	xxxx	aaaa	0
+cccc	cccc	cccc	0
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/r/nullable_or_empty_column.result'
--- plugin/cloud_storage/tests/r/nullable_or_empty_column.result	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/r/nullable_or_empty_column.result	2010-08-19 16:21:41 +0000
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a int primary key, b blob, c int) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+DELETE FROM t1;
+INSERT INTO t1 (a,b) VALUES (222, "bbb");
+INSERT INTO t1 (a,b) VALUES (333, "");
+SELECT * FROM t1;
+a	b	c
+222	bbb	NULL
+333		NULL
+DELETE FROM t1;
+DROP TABLE t1;

=== added directory 'plugin/cloud_storage/tests/t'
=== added file 'plugin/cloud_storage/tests/t/basic_create_table-master.opt'
--- plugin/cloud_storage/tests/t/basic_create_table-master.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_create_table-master.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/basic_create_table.test'
--- plugin/cloud_storage/tests/t/basic_create_table.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_create_table.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,31 @@
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+# test requires a primary key
+--error 1173
+CREATE TABLE t1 (a varchar(20), b int not null)  ENGINE=cloud_storage;
+
+# test too many key parts specified
+--error 1070
+CREATE TABLE t1 (a varchar(20), b int not null, constraint c primary key (a, b))  ENGINE=cloud_storage;
+
+# test has invalid service
+--error 1698
+CREATE TABLE t1 (a varchar(20) primary key, b int not null)  ENGINE=cloud_storage SERVICE=unknow;
+
+# test specified key was too long
+--error 1071
+CREATE TABLE t1 (a varchar(342) primary key, b int not null)  ENGINE=cloud_storage;
+
+CREATE TABLE t1 (a varchar(20) primary key, b int not null)  ENGINE=cloud_storage;
+CREATE TABLE t2 (a varchar(20) primary key,b int not null, c blob not null)  ENGINE=cloud_storage;
+CREATE TABLE t3 (a varchar(341) primary key)  ENGINE=cloud_storage;
+CREATE TABLE t4 (a int primary key)  ENGINE=cloud_storage;
+CREATE TABLE t5 (a int primary key)  ENGINE=cloud_storage SERVICE=awss3;
+
+DROP TABLE t1;
+DROP TABLE t2;
+DROP TABLE t3;
+DROP TABLE t4;
+DROP TABLE t5;

=== added file 'plugin/cloud_storage/tests/t/basic_delete-master.opt'
--- plugin/cloud_storage/tests/t/basic_delete-master.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_delete-master.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/basic_delete.test'
--- plugin/cloud_storage/tests/t/basic_delete.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_delete.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,29 @@
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+
+DELETE FROM t1;
+
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (444, "ddd");
+INSERT INTO t1 VALUES (555, "eee");
+
+SELECT * FROM t1;
+
+DELETE FROM t1 WHERE a>=444;
+DELETE FROM t1 WHERE b="bbb";
+
+SELECT * FROM t1;
+
+DELETE FROM t1 WHERE a!=111;
+
+SELECT * FROM t1;
+
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/t/basic_insert-master.opt'
--- plugin/cloud_storage/tests/t/basic_insert-master.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_insert-master.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/basic_insert.test'
--- plugin/cloud_storage/tests/t/basic_insert.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_insert.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,26 @@
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+
+DELETE FROM t1;
+
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (333, "ccc");
+
+SELECT * FROM t1;
+
+DELETE FROM t1;
+
+INSERT INTO t1 VALUES (111, "aaa");
+
+SELECT * FROM t1;
+
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/t/basic_select-master.opt'
--- plugin/cloud_storage/tests/t/basic_select-master.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_select-master.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/basic_select.test'
--- plugin/cloud_storage/tests/t/basic_select.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_select.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,21 @@
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+
+DELETE FROM t1;
+
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (444, "ddd");
+INSERT INTO t1 VALUES (555, "eee");
+
+SELECT * FROM t1;
+
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/t/basic_update-master.opt'
--- plugin/cloud_storage/tests/t/basic_update-master.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_update-master.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/basic_update.test'
--- plugin/cloud_storage/tests/t/basic_update.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/basic_update.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,28 @@
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (a int primary key, b blob not null) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+
+DELETE FROM t1;
+
+INSERT INTO t1 VALUES (111, "aaa");
+INSERT INTO t1 VALUES (222, "bbb");
+INSERT INTO t1 VALUES (333, "ccc");
+
+# test update key
+UPDATE t1 SET a=444 WHERE a=222;
+UPDATE t1 SET a=555 WHERE b="bbb";
+
+# test update none-key field
+UPDATE t1 SET b="ddd" WHERE a=111;
+
+SELECT * FROM t1;
+
+DELETE FROM t1;
+DROP TABLE t1;

=== added file 'plugin/cloud_storage/tests/t/multi_column-master.opt'
--- plugin/cloud_storage/tests/t/multi_column-master.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/multi_column-master.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/multi_column.test'
--- plugin/cloud_storage/tests/t/multi_column.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/multi_column.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,41 @@
+# Basic test routine for the Cloud Engine
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+# Test multi-column table
+
+CREATE TABLE t1 (
+s3id varchar(20) not null primary key, 
+s3val1 blob not null, 
+s3val2 blob not null, 
+s3val3 int not null default 0) 
+engine=CLOUD_STORAGE 
+BUCKET="pku-cactus_cactus" 
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ" 
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV" 
+SERVICE="awss3";
+
+DELETE FROM t1;
+
+# Test Insert
+
+INSERT INTO t1 VALUES ("aaaa", "aaaa", "aaaa", 0);
+INSERT INTO t1 VALUES ("bbbb", "bbbb", "bbbb", 1);
+INSERT INTO t1 VALUES ("cccc", "cccc", "cccc", 0);
+
+# Test Update
+UPDATE t1 SET s3val1 = "xxxx" WHERE s3id = "aaaa";
+
+# Test Delete
+DELETE FROM t1 WHERE s3id = "bbbb";
+
+# Test Select
+SELECT * FROM t1 WHERE s3id = "cccc";
+SELECT * FROM t1;
+
+DELETE FROM t1;
+
+DROP TABLE t1;
+

=== added file 'plugin/cloud_storage/tests/t/nullable_or_empty_column.opt'
--- plugin/cloud_storage/tests/t/nullable_or_empty_column.opt	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/nullable_or_empty_column.opt	2010-08-19 16:21:41 +0000
@@ -0,0 +1,1 @@
+--plugin_add=cloud_storage

=== added file 'plugin/cloud_storage/tests/t/nullable_or_empty_column.test'
--- plugin/cloud_storage/tests/t/nullable_or_empty_column.test	1970-01-01 00:00:00 +0000
+++ plugin/cloud_storage/tests/t/nullable_or_empty_column.test	2010-08-19 16:21:41 +0000
@@ -0,0 +1,22 @@
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (a int primary key, b blob, c int) 
+ENGINE=cloud_storage
+BUCKET="pku-cactus_cactus"
+KEY_ID="AKIAJOZX7SPNWPXWXHVQ"
+KEY_VALUE="i+v6L7Vu9jsTWKq0Qr6Q4cbX768JcedgGl21IigV"
+SERVICE="awss3";
+
+DELETE FROM t1;
+
+# has null column
+INSERT INTO t1 (a,b) VALUES (222, "bbb");
+# has null and empty column
+INSERT INTO t1 (a,b) VALUES (333, "");
+
+SELECT * FROM t1;
+
+DELETE FROM t1;
+DROP TABLE t1;