Merge lp:~themue/juju-core/027-http-synctools into lp:~go-bot/juju-core/trunk

[Frank Mueller]

synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://codereview.appspot.com/10403043/

synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://codereview.appspot.com/10403043/

[Frank Mueller (themue)]

Reviewers: mp+170154_code.launchpad.net,

Message:
Please take a look.

Description:
synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://code.launchpad.net/~themue/juju-core/027-http-synctools/+merge/170154

(do not edit description out of merge proposal)

Please review this at https://codereview.appspot.com/10403043/

Affected files:
   A [revision details]
   M cmd/juju/synctools.go
   M cmd/juju/synctools_test.go

[John A Meinel (jameinel)]

LGTM

https://codereview.appspot.com/10403043/

=== modified file 'cmd/juju/synctools.go'
--- cmd/juju/synctools.go	2013-05-02 15:55:42 +0000
+++ cmd/juju/synctools.go	2013-06-19 09:02:38 +0000
@@ -5,12 +5,20 @@
 
 import (
 	"bytes"
+	"encoding/xml"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
 	"launchpad.net/gnuflag"
 	"launchpad.net/juju-core/cmd"
 	"launchpad.net/juju-core/environs"
 	"launchpad.net/juju-core/environs/tools"
+	"launchpad.net/juju-core/errors"
 	"launchpad.net/juju-core/log"
 	"launchpad.net/juju-core/state"
 	"launchpad.net/juju-core/version"
@@ -110,11 +118,7 @@
 }
 
 func (c *SyncToolsCommand) Run(ctx *cmd.Context) error {
-	sourceEnv, err := environs.NewFromAttrs(officialBucketAttrs)
-	if err != nil {
-		log.Errorf("failed to initialize the official bucket environment")
-		return err
-	}
+	sourceStorage := newHttpToolsReader()
 	targetEnv, err := environs.NewFromName(c.EnvName)
 	if err != nil {
 		log.Errorf("unable to read %q from environment", c.EnvName)
@@ -123,7 +127,6 @@
 
 	fmt.Fprintf(ctx.Stderr, "listing the source bucket\n")
 	majorVersion := version.Current.Major
-	sourceStorage := sourceEnv.PublicStorage()
 	sourceTools, err := tools.ReadList(sourceStorage, majorVersion)
 	if err != nil {
 		return err
@@ -179,3 +182,96 @@
 	fmt.Fprintf(ctx.Stderr, "copied %d tools\n", len(missing))
 	return nil
 }
+
+// defaultToolsUrl leads to the juju distribution on S3.
+var defaultToolsLocation string = "https://juju-dist.s3.amazonaws.com/"
+
+// listBucketResult is the top level XML element of the storage index.
+type listBucketResult struct {
+	XMLName     xml.Name `xml: "ListBucketResult"`
+	Name        string
+	Prefix      string
+	Marker      string
+	MaxKeys     int
+	IsTruncated bool
+	Contents    []*contents
+}
+
+// content describes one entry of the storage index.
+type contents struct {
+	XMLName      xml.Name `xml: "Contents"`
+	Key          string
+	LastModified time.Time
+	ETag         string
+	Size         int
+	StorageClass string
+}
+
+// httpToolsReader implements the environs.StorageReader interface by
+// accessing the juju-core public store simply using http.
+type httpToolsReader struct {
+	location string
+}
+
+// newHttpToolsReader creates a storage reader for the http 
+// access to the juju-core public store.
+func newHttpToolsReader() environs.StorageReader {
+	return &httpToolsReader{defaultToolsLocation}
+}
+
+// Get opens the given storage file and returns a ReadCloser
+// that can be used to read its contents.
+func (h *httpToolsReader) Get(name string) (io.ReadCloser, error) {
+	locationName, err := h.URL(name)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.Get(locationName)
+	if err != nil && resp.StatusCode == http.StatusNotFound {
+		return nil, &errors.NotFoundError{err, ""}
+	}
+	return resp.Body, nil
+}
+
+// List lists all names in the storage with the given prefix.
+func (h *httpToolsReader) List(prefix string) ([]string, error) {
+	lbr, err := h.getListBucketResult()
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, c := range lbr.Contents {
+		if strings.HasPrefix(c.Key, prefix) {
+			names = append(names, c.Key)
+		}
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// URL returns a URL that can be used to access the given storage file.
+func (h *httpToolsReader) URL(name string) (string, error) {
+	if strings.HasSuffix(h.location, "/") {
+		return h.location + name, nil
+	}
+	return h.location + "/" + name, nil
+}
+
+// getListBucketResult retrieves the index of the storage,
+func (h *httpToolsReader) getListBucketResult() (*listBucketResult, error) {
+	resp, err := http.Get(h.location)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	buf, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var lbr listBucketResult
+	err = xml.Unmarshal(buf, &lbr)
+	if err != nil {
+		return nil, err
+	}
+	return &lbr, nil
+}
 
=== modified file 'cmd/juju/synctools_test.go'
--- cmd/juju/synctools_test.go	2013-05-02 15:55:42 +0000
+++ cmd/juju/synctools_test.go	2013-06-19 09:02:38 +0000
@@ -4,6 +4,15 @@
 package main
 
 import (
+	"encoding/xml"
+	"fmt"
+	"hash/crc32"
+	"net"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
 	. "launchpad.net/gocheck"
 	"launchpad.net/juju-core/cmd"
 	"launchpad.net/juju-core/environs"
@@ -16,10 +25,12 @@
 
 type syncToolsSuite struct {
 	testing.LoggingSuite
-	home        *testing.FakeHome
-	targetEnv   environs.Environ
-	origAttrs   map[string]interface{}
-	origVersion version.Binary
+	home         *testing.FakeHome
+	targetEnv    environs.Environ
+	origVersion  version.Binary
+	origLocation string
+	listener     net.Listener
+	storage      *testStorage
 }
 
 func (s *syncToolsSuite) SetUpTest(c *C) {
@@ -42,30 +53,20 @@
 	envtesting.RemoveAllTools(c, s.targetEnv)
 
 	// Create a source environment and populate its public tools.
-	dummyAttrs := map[string]interface{}{
-		"name":         "test-source",
-		"type":         "dummy",
-		"state-server": false,
-		// Note: Without this, you get "no public ssh keys found", which seems
-		// a bit odd for the "dummy" environment
-		"authorized-keys": "I-am-not-a-real-key",
-	}
-	env, err := environs.NewFromAttrs(dummyAttrs)
+	s.listener, s.storage, err = listen("127.0.0.1", 0)
 	c.Assert(err, IsNil)
-	c.Assert(env, NotNil)
-	envtesting.RemoveAllTools(c, env)
-	store := env.PublicStorage().(environs.Storage)
+
 	for _, vers := range vAll {
-		envtesting.UploadFakeToolsVersion(c, store, vers)
+		s.storage.putFakeToolsVersion(vers)
 	}
-	// Overwrite the official source bucket to the new dummy 'test-source',
-	// saving the original value for cleanup
-	s.origAttrs = officialBucketAttrs
-	officialBucketAttrs = dummyAttrs
+
+	s.origLocation = defaultToolsLocation
+	defaultToolsLocation = s.storage.location
 }
 
 func (s *syncToolsSuite) TearDownTest(c *C) {
-	officialBucketAttrs = s.origAttrs
+	c.Assert(s.listener.Close(), IsNil)
+	defaultToolsLocation = s.origLocation
 	dummy.Reset()
 	s.home.Restore()
 	version.Current = s.origVersion
@@ -190,3 +191,107 @@
 	v200p64 = version.MustParseBinary("2.0.0-precise-amd64")
 	vAll    = append(v1all, v200p64)
 )
+
+// testStorage acts like the juju distribution storage at S3 
+// to provide the juju tools.
+type testStorage struct {
+	location string
+	files    map[string][]byte
+}
+
+// ServeHTTP handles the HTTP requests to the storage mock.
+func (t *testStorage) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	switch req.Method {
+	case "GET":
+		if req.URL.Path == "/" {
+			t.handleIndex(w, req)
+		} else {
+			t.handleGet(w, req)
+		}
+	default:
+		http.Error(w, "method "+req.Method+" is not supported", http.StatusMethodNotAllowed)
+	}
+}
+
+// handleIndex returns the index XML file to the client.
+func (t *testStorage) handleIndex(w http.ResponseWriter, req *http.Request) {
+	lbr := &listBucketResult{
+		Name:        "juju-dist",
+		Prefix:      "",
+		Marker:      "",
+		MaxKeys:     1000,
+		IsTruncated: false,
+	}
+	names := []string{}
+	for name := range t.files {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		h := crc32.NewIEEE()
+		h.Write([]byte(t.files[name]))
+		contents := &contents{
+			Key:          name,
+			LastModified: time.Now(),
+			ETag:         fmt.Sprintf("%x", h.Sum(nil)),
+			Size:         len([]byte(t.files[name])),
+			StorageClass: "STANDARD",
+		}
+		lbr.Contents = append(lbr.Contents, contents)
+	}
+	buf, err := xml.Marshal(lbr)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("500 %v", err), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/xml")
+	w.Write(buf)
+}
+
+// handleGet returns a storage file to the client.
+func (t *testStorage) handleGet(w http.ResponseWriter, req *http.Request) {
+	data, ok := t.files[req.URL.Path]
+	if !ok {
+		http.Error(w, "404 file not found", http.StatusNotFound)
+		return
+	}
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Write(data)
+}
+
+// putFakeToolsVersion stores a faked binary in the tools storage.
+func (t *testStorage) putFakeToolsVersion(vers version.Binary) {
+	data := vers.String()
+	name := tools.StorageName(vers)
+	parts := strings.Split(name, "/")
+	if len(parts) > 1 {
+		// Also create paths as entries. Needed for
+		// the correct contents of the list bucket result.
+		path := ""
+		for i := 0; i < len(parts)-1; i++ {
+			path = path + parts[i] + "/"
+			t.files[path] = []byte{}
+		}
+	}
+	t.files[name] = []byte(data)
+}
+
+// listen starts an HTTP listener to serve the
+// provider storage.
+func listen(ip string, port int) (net.Listener, *testStorage, error) {
+	storage := &testStorage{
+		files: make(map[string][]byte),
+	}
+	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot start listener: %v", err)
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/", storage)
+
+	go http.Serve(listener, mux)
+
+	storage.location = fmt.Sprintf("http://%s:%d/", ip, listener.Addr().(*net.TCPAddr).Port)
+
+	return listener, storage, nil
+}