Merge lp:~themue/juju-core/027-http-synctools into lp:~go-bot/juju-core/trunk

Proposed by Frank Mueller
Status: Merged
Approved by: Frank Mueller
Approved revision: 1283
Merged at revision: 1303
Proposed branch: lp:~themue/juju-core/027-http-synctools
Merge into: lp:~go-bot/juju-core/trunk
Diff against target: 326 lines (+229/-28)
2 files modified
cmd/juju/synctools.go (+102/-6)
cmd/juju/synctools_test.go (+127/-22)
To merge this branch: bzr merge lp:~themue/juju-core/027-http-synctools
Reviewer Review Type Date Requested Status
Juju Engineering Pending
Review via email: mp+170154@code.launchpad.net

Commit message

synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://codereview.appspot.com/10403043/

Description of the change

synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://codereview.appspot.com/10403043/

To post a comment you must log in.
Revision history for this message
Frank Mueller (themue) wrote :

Reviewers: mp+170154_code.launchpad.net,

Message:
Please take a look.

Description:
synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://code.launchpad.net/~themue/juju-core/027-http-synctools/+merge/170154

(do not edit description out of merge proposal)

Please review this at https://codereview.appspot.com/10403043/

Affected files:
   A [revision details]
   M cmd/juju/synctools.go
   M cmd/juju/synctools_test.go

Revision history for this message
John A Meinel (jameinel) wrote :
1283. By Frank Mueller

synctools: merged trunk

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'cmd/juju/synctools.go'
--- cmd/juju/synctools.go 2013-05-02 15:55:42 +0000
+++ cmd/juju/synctools.go 2013-06-19 09:02:38 +0000
@@ -5,12 +5,20 @@
55
6import (6import (
7 "bytes"7 "bytes"
8 "encoding/xml"
8 "fmt"9 "fmt"
9 "io"10 "io"
11 "io/ioutil"
12 "net/http"
13 "sort"
14 "strings"
15 "time"
16
10 "launchpad.net/gnuflag"17 "launchpad.net/gnuflag"
11 "launchpad.net/juju-core/cmd"18 "launchpad.net/juju-core/cmd"
12 "launchpad.net/juju-core/environs"19 "launchpad.net/juju-core/environs"
13 "launchpad.net/juju-core/environs/tools"20 "launchpad.net/juju-core/environs/tools"
21 "launchpad.net/juju-core/errors"
14 "launchpad.net/juju-core/log"22 "launchpad.net/juju-core/log"
15 "launchpad.net/juju-core/state"23 "launchpad.net/juju-core/state"
16 "launchpad.net/juju-core/version"24 "launchpad.net/juju-core/version"
@@ -110,11 +118,7 @@
110}118}
111119
112func (c *SyncToolsCommand) Run(ctx *cmd.Context) error {120func (c *SyncToolsCommand) Run(ctx *cmd.Context) error {
113 sourceEnv, err := environs.NewFromAttrs(officialBucketAttrs)121 sourceStorage := newHttpToolsReader()
114 if err != nil {
115 log.Errorf("failed to initialize the official bucket environment")
116 return err
117 }
118 targetEnv, err := environs.NewFromName(c.EnvName)122 targetEnv, err := environs.NewFromName(c.EnvName)
119 if err != nil {123 if err != nil {
120 log.Errorf("unable to read %q from environment", c.EnvName)124 log.Errorf("unable to read %q from environment", c.EnvName)
@@ -123,7 +127,6 @@
123127
124 fmt.Fprintf(ctx.Stderr, "listing the source bucket\n")128 fmt.Fprintf(ctx.Stderr, "listing the source bucket\n")
125 majorVersion := version.Current.Major129 majorVersion := version.Current.Major
126 sourceStorage := sourceEnv.PublicStorage()
127 sourceTools, err := tools.ReadList(sourceStorage, majorVersion)130 sourceTools, err := tools.ReadList(sourceStorage, majorVersion)
128 if err != nil {131 if err != nil {
129 return err132 return err
@@ -179,3 +182,96 @@
179 fmt.Fprintf(ctx.Stderr, "copied %d tools\n", len(missing))182 fmt.Fprintf(ctx.Stderr, "copied %d tools\n", len(missing))
180 return nil183 return nil
181}184}
185
186// defaultToolsUrl leads to the juju distribution on S3.
187var defaultToolsLocation string = "https://juju-dist.s3.amazonaws.com/"
188
189// listBucketResult is the top level XML element of the storage index.
190type listBucketResult struct {
191 XMLName xml.Name `xml: "ListBucketResult"`
192 Name string
193 Prefix string
194 Marker string
195 MaxKeys int
196 IsTruncated bool
197 Contents []*contents
198}
199
200// content describes one entry of the storage index.
201type contents struct {
202 XMLName xml.Name `xml: "Contents"`
203 Key string
204 LastModified time.Time
205 ETag string
206 Size int
207 StorageClass string
208}
209
210// httpToolsReader implements the environs.StorageReader interface by
211// accessing the juju-core public store simply using http.
212type httpToolsReader struct {
213 location string
214}
215
216// newHttpToolsReader creates a storage reader for the http
217// access to the juju-core public store.
218func newHttpToolsReader() environs.StorageReader {
219 return &httpToolsReader{defaultToolsLocation}
220}
221
222// Get opens the given storage file and returns a ReadCloser
223// that can be used to read its contents.
224func (h *httpToolsReader) Get(name string) (io.ReadCloser, error) {
225 locationName, err := h.URL(name)
226 if err != nil {
227 return nil, err
228 }
229 resp, err := http.Get(locationName)
230 if err != nil && resp.StatusCode == http.StatusNotFound {
231 return nil, &errors.NotFoundError{err, ""}
232 }
233 return resp.Body, nil
234}
235
236// List lists all names in the storage with the given prefix.
237func (h *httpToolsReader) List(prefix string) ([]string, error) {
238 lbr, err := h.getListBucketResult()
239 if err != nil {
240 return nil, err
241 }
242 var names []string
243 for _, c := range lbr.Contents {
244 if strings.HasPrefix(c.Key, prefix) {
245 names = append(names, c.Key)
246 }
247 }
248 sort.Strings(names)
249 return names, nil
250}
251
252// URL returns a URL that can be used to access the given storage file.
253func (h *httpToolsReader) URL(name string) (string, error) {
254 if strings.HasSuffix(h.location, "/") {
255 return h.location + name, nil
256 }
257 return h.location + "/" + name, nil
258}
259
260// getListBucketResult retrieves the index of the storage,
261func (h *httpToolsReader) getListBucketResult() (*listBucketResult, error) {
262 resp, err := http.Get(h.location)
263 if err != nil {
264 return nil, err
265 }
266 defer resp.Body.Close()
267 buf, err := ioutil.ReadAll(resp.Body)
268 if err != nil {
269 return nil, err
270 }
271 var lbr listBucketResult
272 err = xml.Unmarshal(buf, &lbr)
273 if err != nil {
274 return nil, err
275 }
276 return &lbr, nil
277}
182278
=== modified file 'cmd/juju/synctools_test.go'
--- cmd/juju/synctools_test.go 2013-05-02 15:55:42 +0000
+++ cmd/juju/synctools_test.go 2013-06-19 09:02:38 +0000
@@ -4,6 +4,15 @@
4package main4package main
55
6import (6import (
7 "encoding/xml"
8 "fmt"
9 "hash/crc32"
10 "net"
11 "net/http"
12 "sort"
13 "strings"
14 "time"
15
7 . "launchpad.net/gocheck"16 . "launchpad.net/gocheck"
8 "launchpad.net/juju-core/cmd"17 "launchpad.net/juju-core/cmd"
9 "launchpad.net/juju-core/environs"18 "launchpad.net/juju-core/environs"
@@ -16,10 +25,12 @@
1625
17type syncToolsSuite struct {26type syncToolsSuite struct {
18 testing.LoggingSuite27 testing.LoggingSuite
19 home *testing.FakeHome28 home *testing.FakeHome
20 targetEnv environs.Environ29 targetEnv environs.Environ
21 origAttrs map[string]interface{}30 origVersion version.Binary
22 origVersion version.Binary31 origLocation string
32 listener net.Listener
33 storage *testStorage
23}34}
2435
25func (s *syncToolsSuite) SetUpTest(c *C) {36func (s *syncToolsSuite) SetUpTest(c *C) {
@@ -42,30 +53,20 @@
42 envtesting.RemoveAllTools(c, s.targetEnv)53 envtesting.RemoveAllTools(c, s.targetEnv)
4354
44 // Create a source environment and populate its public tools.55 // Create a source environment and populate its public tools.
45 dummyAttrs := map[string]interface{}{56 s.listener, s.storage, err = listen("127.0.0.1", 0)
46 "name": "test-source",
47 "type": "dummy",
48 "state-server": false,
49 // Note: Without this, you get "no public ssh keys found", which seems
50 // a bit odd for the "dummy" environment
51 "authorized-keys": "I-am-not-a-real-key",
52 }
53 env, err := environs.NewFromAttrs(dummyAttrs)
54 c.Assert(err, IsNil)57 c.Assert(err, IsNil)
55 c.Assert(env, NotNil)58
56 envtesting.RemoveAllTools(c, env)
57 store := env.PublicStorage().(environs.Storage)
58 for _, vers := range vAll {59 for _, vers := range vAll {
59 envtesting.UploadFakeToolsVersion(c, store, vers)60 s.storage.putFakeToolsVersion(vers)
60 }61 }
61 // Overwrite the official source bucket to the new dummy 'test-source',62
62 // saving the original value for cleanup63 s.origLocation = defaultToolsLocation
63 s.origAttrs = officialBucketAttrs64 defaultToolsLocation = s.storage.location
64 officialBucketAttrs = dummyAttrs
65}65}
6666
67func (s *syncToolsSuite) TearDownTest(c *C) {67func (s *syncToolsSuite) TearDownTest(c *C) {
68 officialBucketAttrs = s.origAttrs68 c.Assert(s.listener.Close(), IsNil)
69 defaultToolsLocation = s.origLocation
69 dummy.Reset()70 dummy.Reset()
70 s.home.Restore()71 s.home.Restore()
71 version.Current = s.origVersion72 version.Current = s.origVersion
@@ -190,3 +191,107 @@
190 v200p64 = version.MustParseBinary("2.0.0-precise-amd64")191 v200p64 = version.MustParseBinary("2.0.0-precise-amd64")
191 vAll = append(v1all, v200p64)192 vAll = append(v1all, v200p64)
192)193)
194
195// testStorage acts like the juju distribution storage at S3
196// to provide the juju tools.
197type testStorage struct {
198 location string
199 files map[string][]byte
200}
201
202// ServeHTTP handles the HTTP requests to the storage mock.
203func (t *testStorage) ServeHTTP(w http.ResponseWriter, req *http.Request) {
204 switch req.Method {
205 case "GET":
206 if req.URL.Path == "/" {
207 t.handleIndex(w, req)
208 } else {
209 t.handleGet(w, req)
210 }
211 default:
212 http.Error(w, "method "+req.Method+" is not supported", http.StatusMethodNotAllowed)
213 }
214}
215
216// handleIndex returns the index XML file to the client.
217func (t *testStorage) handleIndex(w http.ResponseWriter, req *http.Request) {
218 lbr := &listBucketResult{
219 Name: "juju-dist",
220 Prefix: "",
221 Marker: "",
222 MaxKeys: 1000,
223 IsTruncated: false,
224 }
225 names := []string{}
226 for name := range t.files {
227 names = append(names, name)
228 }
229 sort.Strings(names)
230 for _, name := range names {
231 h := crc32.NewIEEE()
232 h.Write([]byte(t.files[name]))
233 contents := &contents{
234 Key: name,
235 LastModified: time.Now(),
236 ETag: fmt.Sprintf("%x", h.Sum(nil)),
237 Size: len([]byte(t.files[name])),
238 StorageClass: "STANDARD",
239 }
240 lbr.Contents = append(lbr.Contents, contents)
241 }
242 buf, err := xml.Marshal(lbr)
243 if err != nil {
244 http.Error(w, fmt.Sprintf("500 %v", err), http.StatusInternalServerError)
245 return
246 }
247 w.Header().Set("Content-Type", "application/xml")
248 w.Write(buf)
249}
250
251// handleGet returns a storage file to the client.
252func (t *testStorage) handleGet(w http.ResponseWriter, req *http.Request) {
253 data, ok := t.files[req.URL.Path]
254 if !ok {
255 http.Error(w, "404 file not found", http.StatusNotFound)
256 return
257 }
258 w.Header().Set("Content-Type", "application/octet-stream")
259 w.Write(data)
260}
261
262// putFakeToolsVersion stores a faked binary in the tools storage.
263func (t *testStorage) putFakeToolsVersion(vers version.Binary) {
264 data := vers.String()
265 name := tools.StorageName(vers)
266 parts := strings.Split(name, "/")
267 if len(parts) > 1 {
268 // Also create paths as entries. Needed for
269 // the correct contents of the list bucket result.
270 path := ""
271 for i := 0; i < len(parts)-1; i++ {
272 path = path + parts[i] + "/"
273 t.files[path] = []byte{}
274 }
275 }
276 t.files[name] = []byte(data)
277}
278
279// listen starts an HTTP listener to serve the
280// provider storage.
281func listen(ip string, port int) (net.Listener, *testStorage, error) {
282 storage := &testStorage{
283 files: make(map[string][]byte),
284 }
285 listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
286 if err != nil {
287 return nil, nil, fmt.Errorf("cannot start listener: %v", err)
288 }
289 mux := http.NewServeMux()
290 mux.Handle("/", storage)
291
292 go http.Serve(listener, mux)
293
294 storage.location = fmt.Sprintf("http://%s:%d/", ip, listener.Addr().(*net.TCPAddr).Port)
295
296 return listener, storage, nil
297}

Subscribers

People subscribed via source and target branches

to status/vote changes: