Merge lp:~themue/juju-core/027-http-synctools into lp:~go-bot/juju-core/trunk

Proposed by Frank Mueller on 2013-06-18
Status: Merged
Approved by: Frank Mueller on 2013-06-19
Approved revision: 1283
Merged at revision: 1303
Proposed branch: lp:~themue/juju-core/027-http-synctools
Merge into: lp:~go-bot/juju-core/trunk
Diff against target: 326 lines (+229/-28)
2 files modified
cmd/juju/synctools.go (+102/-6)
cmd/juju/synctools_test.go (+127/-22)
To merge this branch: bzr merge lp:~themue/juju-core/027-http-synctools
Reviewer Review Type Date Requested Status
Juju Engineering 2013-06-18 Pending
Review via email: mp+170154@code.launchpad.net

Commit message

synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://codereview.appspot.com/10403043/

Description of the change

synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://codereview.appspot.com/10403043/

To post a comment you must log in.
Frank Mueller (themue) wrote :

Reviewers: mp+170154_code.launchpad.net,

Message:
Please take a look.

Description:
synctools: juju-dist access now via https

Instead of using an EC2 environment especially for
the juju-dist storage now a simple access via https
is used. This way no AWS credentials are needed.

https://code.launchpad.net/~themue/juju-core/027-http-synctools/+merge/170154

(do not edit description out of merge proposal)

Please review this at https://codereview.appspot.com/10403043/

Affected files:
   A [revision details]
   M cmd/juju/synctools.go
   M cmd/juju/synctools_test.go

1283. By Frank Mueller on 2013-06-19

synctools: merged trunk

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'cmd/juju/synctools.go'
2--- cmd/juju/synctools.go 2013-05-02 15:55:42 +0000
3+++ cmd/juju/synctools.go 2013-06-19 09:02:38 +0000
4@@ -5,12 +5,20 @@
5
6 import (
7 "bytes"
8+ "encoding/xml"
9 "fmt"
10 "io"
11+ "io/ioutil"
12+ "net/http"
13+ "sort"
14+ "strings"
15+ "time"
16+
17 "launchpad.net/gnuflag"
18 "launchpad.net/juju-core/cmd"
19 "launchpad.net/juju-core/environs"
20 "launchpad.net/juju-core/environs/tools"
21+ "launchpad.net/juju-core/errors"
22 "launchpad.net/juju-core/log"
23 "launchpad.net/juju-core/state"
24 "launchpad.net/juju-core/version"
25@@ -110,11 +118,7 @@
26 }
27
28 func (c *SyncToolsCommand) Run(ctx *cmd.Context) error {
29- sourceEnv, err := environs.NewFromAttrs(officialBucketAttrs)
30- if err != nil {
31- log.Errorf("failed to initialize the official bucket environment")
32- return err
33- }
34+ sourceStorage := newHttpToolsReader()
35 targetEnv, err := environs.NewFromName(c.EnvName)
36 if err != nil {
37 log.Errorf("unable to read %q from environment", c.EnvName)
38@@ -123,7 +127,6 @@
39
40 fmt.Fprintf(ctx.Stderr, "listing the source bucket\n")
41 majorVersion := version.Current.Major
42- sourceStorage := sourceEnv.PublicStorage()
43 sourceTools, err := tools.ReadList(sourceStorage, majorVersion)
44 if err != nil {
45 return err
46@@ -179,3 +182,96 @@
47 fmt.Fprintf(ctx.Stderr, "copied %d tools\n", len(missing))
48 return nil
49 }
50+
51+// defaultToolsUrl leads to the juju distribution on S3.
52+var defaultToolsLocation string = "https://juju-dist.s3.amazonaws.com/"
53+
54+// listBucketResult is the top level XML element of the storage index.
55+type listBucketResult struct {
56+ XMLName xml.Name `xml: "ListBucketResult"`
57+ Name string
58+ Prefix string
59+ Marker string
60+ MaxKeys int
61+ IsTruncated bool
62+ Contents []*contents
63+}
64+
65+// content describes one entry of the storage index.
66+type contents struct {
67+ XMLName xml.Name `xml: "Contents"`
68+ Key string
69+ LastModified time.Time
70+ ETag string
71+ Size int
72+ StorageClass string
73+}
74+
75+// httpToolsReader implements the environs.StorageReader interface by
76+// accessing the juju-core public store simply using http.
77+type httpToolsReader struct {
78+ location string
79+}
80+
81+// newHttpToolsReader creates a storage reader for the http
82+// access to the juju-core public store.
83+func newHttpToolsReader() environs.StorageReader {
84+ return &httpToolsReader{defaultToolsLocation}
85+}
86+
87+// Get opens the given storage file and returns a ReadCloser
88+// that can be used to read its contents.
89+func (h *httpToolsReader) Get(name string) (io.ReadCloser, error) {
90+ locationName, err := h.URL(name)
91+ if err != nil {
92+ return nil, err
93+ }
94+ resp, err := http.Get(locationName)
95+ if err != nil && resp.StatusCode == http.StatusNotFound {
96+ return nil, &errors.NotFoundError{err, ""}
97+ }
98+ return resp.Body, nil
99+}
100+
101+// List lists all names in the storage with the given prefix.
102+func (h *httpToolsReader) List(prefix string) ([]string, error) {
103+ lbr, err := h.getListBucketResult()
104+ if err != nil {
105+ return nil, err
106+ }
107+ var names []string
108+ for _, c := range lbr.Contents {
109+ if strings.HasPrefix(c.Key, prefix) {
110+ names = append(names, c.Key)
111+ }
112+ }
113+ sort.Strings(names)
114+ return names, nil
115+}
116+
117+// URL returns a URL that can be used to access the given storage file.
118+func (h *httpToolsReader) URL(name string) (string, error) {
119+ if strings.HasSuffix(h.location, "/") {
120+ return h.location + name, nil
121+ }
122+ return h.location + "/" + name, nil
123+}
124+
125+// getListBucketResult retrieves the index of the storage,
126+func (h *httpToolsReader) getListBucketResult() (*listBucketResult, error) {
127+ resp, err := http.Get(h.location)
128+ if err != nil {
129+ return nil, err
130+ }
131+ defer resp.Body.Close()
132+ buf, err := ioutil.ReadAll(resp.Body)
133+ if err != nil {
134+ return nil, err
135+ }
136+ var lbr listBucketResult
137+ err = xml.Unmarshal(buf, &lbr)
138+ if err != nil {
139+ return nil, err
140+ }
141+ return &lbr, nil
142+}
143
144=== modified file 'cmd/juju/synctools_test.go'
145--- cmd/juju/synctools_test.go 2013-05-02 15:55:42 +0000
146+++ cmd/juju/synctools_test.go 2013-06-19 09:02:38 +0000
147@@ -4,6 +4,15 @@
148 package main
149
150 import (
151+ "encoding/xml"
152+ "fmt"
153+ "hash/crc32"
154+ "net"
155+ "net/http"
156+ "sort"
157+ "strings"
158+ "time"
159+
160 . "launchpad.net/gocheck"
161 "launchpad.net/juju-core/cmd"
162 "launchpad.net/juju-core/environs"
163@@ -16,10 +25,12 @@
164
165 type syncToolsSuite struct {
166 testing.LoggingSuite
167- home *testing.FakeHome
168- targetEnv environs.Environ
169- origAttrs map[string]interface{}
170- origVersion version.Binary
171+ home *testing.FakeHome
172+ targetEnv environs.Environ
173+ origVersion version.Binary
174+ origLocation string
175+ listener net.Listener
176+ storage *testStorage
177 }
178
179 func (s *syncToolsSuite) SetUpTest(c *C) {
180@@ -42,30 +53,20 @@
181 envtesting.RemoveAllTools(c, s.targetEnv)
182
183 // Create a source environment and populate its public tools.
184- dummyAttrs := map[string]interface{}{
185- "name": "test-source",
186- "type": "dummy",
187- "state-server": false,
188- // Note: Without this, you get "no public ssh keys found", which seems
189- // a bit odd for the "dummy" environment
190- "authorized-keys": "I-am-not-a-real-key",
191- }
192- env, err := environs.NewFromAttrs(dummyAttrs)
193+ s.listener, s.storage, err = listen("127.0.0.1", 0)
194 c.Assert(err, IsNil)
195- c.Assert(env, NotNil)
196- envtesting.RemoveAllTools(c, env)
197- store := env.PublicStorage().(environs.Storage)
198+
199 for _, vers := range vAll {
200- envtesting.UploadFakeToolsVersion(c, store, vers)
201+ s.storage.putFakeToolsVersion(vers)
202 }
203- // Overwrite the official source bucket to the new dummy 'test-source',
204- // saving the original value for cleanup
205- s.origAttrs = officialBucketAttrs
206- officialBucketAttrs = dummyAttrs
207+
208+ s.origLocation = defaultToolsLocation
209+ defaultToolsLocation = s.storage.location
210 }
211
212 func (s *syncToolsSuite) TearDownTest(c *C) {
213- officialBucketAttrs = s.origAttrs
214+ c.Assert(s.listener.Close(), IsNil)
215+ defaultToolsLocation = s.origLocation
216 dummy.Reset()
217 s.home.Restore()
218 version.Current = s.origVersion
219@@ -190,3 +191,107 @@
220 v200p64 = version.MustParseBinary("2.0.0-precise-amd64")
221 vAll = append(v1all, v200p64)
222 )
223+
224+// testStorage acts like the juju distribution storage at S3
225+// to provide the juju tools.
226+type testStorage struct {
227+ location string
228+ files map[string][]byte
229+}
230+
231+// ServeHTTP handles the HTTP requests to the storage mock.
232+func (t *testStorage) ServeHTTP(w http.ResponseWriter, req *http.Request) {
233+ switch req.Method {
234+ case "GET":
235+ if req.URL.Path == "/" {
236+ t.handleIndex(w, req)
237+ } else {
238+ t.handleGet(w, req)
239+ }
240+ default:
241+ http.Error(w, "method "+req.Method+" is not supported", http.StatusMethodNotAllowed)
242+ }
243+}
244+
245+// handleIndex returns the index XML file to the client.
246+func (t *testStorage) handleIndex(w http.ResponseWriter, req *http.Request) {
247+ lbr := &listBucketResult{
248+ Name: "juju-dist",
249+ Prefix: "",
250+ Marker: "",
251+ MaxKeys: 1000,
252+ IsTruncated: false,
253+ }
254+ names := []string{}
255+ for name := range t.files {
256+ names = append(names, name)
257+ }
258+ sort.Strings(names)
259+ for _, name := range names {
260+ h := crc32.NewIEEE()
261+ h.Write([]byte(t.files[name]))
262+ contents := &contents{
263+ Key: name,
264+ LastModified: time.Now(),
265+ ETag: fmt.Sprintf("%x", h.Sum(nil)),
266+ Size: len([]byte(t.files[name])),
267+ StorageClass: "STANDARD",
268+ }
269+ lbr.Contents = append(lbr.Contents, contents)
270+ }
271+ buf, err := xml.Marshal(lbr)
272+ if err != nil {
273+ http.Error(w, fmt.Sprintf("500 %v", err), http.StatusInternalServerError)
274+ return
275+ }
276+ w.Header().Set("Content-Type", "application/xml")
277+ w.Write(buf)
278+}
279+
280+// handleGet returns a storage file to the client.
281+func (t *testStorage) handleGet(w http.ResponseWriter, req *http.Request) {
282+ data, ok := t.files[req.URL.Path]
283+ if !ok {
284+ http.Error(w, "404 file not found", http.StatusNotFound)
285+ return
286+ }
287+ w.Header().Set("Content-Type", "application/octet-stream")
288+ w.Write(data)
289+}
290+
291+// putFakeToolsVersion stores a faked binary in the tools storage.
292+func (t *testStorage) putFakeToolsVersion(vers version.Binary) {
293+ data := vers.String()
294+ name := tools.StorageName(vers)
295+ parts := strings.Split(name, "/")
296+ if len(parts) > 1 {
297+ // Also create paths as entries. Needed for
298+ // the correct contents of the list bucket result.
299+ path := ""
300+ for i := 0; i < len(parts)-1; i++ {
301+ path = path + parts[i] + "/"
302+ t.files[path] = []byte{}
303+ }
304+ }
305+ t.files[name] = []byte(data)
306+}
307+
308+// listen starts an HTTP listener to serve the
309+// provider storage.
310+func listen(ip string, port int) (net.Listener, *testStorage, error) {
311+ storage := &testStorage{
312+ files: make(map[string][]byte),
313+ }
314+ listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
315+ if err != nil {
316+ return nil, nil, fmt.Errorf("cannot start listener: %v", err)
317+ }
318+ mux := http.NewServeMux()
319+ mux.Handle("/", storage)
320+
321+ go http.Serve(listener, mux)
322+
323+ storage.location = fmt.Sprintf("http://%s:%d/", ip, listener.Addr().(*net.TCPAddr).Port)
324+
325+ return listener, storage, nil
326+}

Subscribers

People subscribed via source and target branches

to status/vote changes: