1
=== removed directory 'cmd/charmd'
2
=== removed file 'cmd/charmd/config.yaml'
3
--- cmd/charmd/config.yaml	2012-04-02 15:05:40 +0000
4
+++ cmd/charmd/config.yaml	1970-01-01 00:00:00 +0000
5
@@ -1,2 +0,0 @@
6
1
mongo-url: localhost:60017
7
2
api-addr: localhost:8080
8
3
0
9
=== removed file 'cmd/charmd/main.go'
10
--- cmd/charmd/main.go	2012-06-21 20:40:39 +0000
11
+++ cmd/charmd/main.go	1970-01-01 00:00:00 +0000
12
@@ -1,74 +0,0 @@
13
1
package main
14
2
15
3
import (
16
4
	"fmt"
17
5
	"io/ioutil"
18
6
	"launchpad.net/goyaml"
19
7
	"launchpad.net/juju-core/log"
20
8
	"launchpad.net/juju-core/store"
21
9
	stdlog "log"
22
10
	"net/http"
23
11
	"os"
24
12
	"path/filepath"
25
13
)
26
14
27
15
func main() {
28
16
	log.Target = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
29
17
	err := serve()
30
18
	if err != nil {
31
19
		fmt.Fprintf(os.Stderr, "%v\n", err)
32
20
		os.Exit(1)
33
21
	}
34
22
}
35
23
36
24
type config struct {
37
25
	MongoURL string `yaml:"mongo-url"`
38
26
	APIAddr  string `yaml:"api-addr"`
39
27
}
40
28
41
29
func readConfig(path string, conf interface{}) error {
42
30
	f, err := os.Open(path)
43
31
	if err != nil {
44
32
		return fmt.Errorf("opening config file: %v", err)
45
33
	}
46
34
	data, err := ioutil.ReadAll(f)
47
35
	f.Close()
48
36
	if err != nil {
49
37
		return fmt.Errorf("reading config file: %v", err)
50
38
	}
51
39
	err = goyaml.Unmarshal(data, conf)
52
40
	if err != nil {
53
41
		return fmt.Errorf("processing config file: %v", err)
54
42
	}
55
43
	return nil
56
44
}
57
45
58
46
func serve() error {
59
47
	var confPath string
60
48
	if len(os.Args) == 2 {
61
49
		if _, err := os.Stat(os.Args[1]); err == nil {
62
50
			confPath = os.Args[1]
63
51
		}
64
52
	}
65
53
	if confPath == "" {
66
54
		return fmt.Errorf("usage: %s <config path>", filepath.Base(os.Args[0]))
67
55
	}
68
56
	var conf config
69
57
	err := readConfig(confPath, &conf)
70
58
	if err != nil {
71
59
		return err
72
60
	}
73
61
	if conf.MongoURL == "" || conf.APIAddr == "" {
74
62
		return fmt.Errorf("missing mongo-url or api-addr in config file")
75
63
	}
76
64
	s, err := store.Open(conf.MongoURL)
77
65
	if err != nil {
78
66
		return err
79
67
	}
80
68
	defer s.Close()
81
69
	server, err := store.NewServer(s)
82
70
	if err != nil {
83
71
		return err
84
72
	}
85
73
	return http.ListenAndServe(conf.APIAddr, server)
86
74
}
87
75
0
88
=== removed directory 'cmd/charmload'
89
=== removed file 'cmd/charmload/config.yaml'
90
--- cmd/charmload/config.yaml	2012-04-02 15:05:40 +0000
91
+++ cmd/charmload/config.yaml	1970-01-01 00:00:00 +0000
92
@@ -1,1 +0,0 @@
93
1
mongo-url: localhost:60017
94
2
0
95
=== removed file 'cmd/charmload/main.go'
96
--- cmd/charmload/main.go	2012-06-21 20:40:39 +0000
97
+++ cmd/charmload/main.go	1970-01-01 00:00:00 +0000
98
@@ -1,75 +0,0 @@
99
1
package main
100
2
101
3
import (
102
4
	"fmt"
103
5
	"io/ioutil"
104
6
	"launchpad.net/goyaml"
105
7
	"launchpad.net/juju-core/log"
106
8
	"launchpad.net/juju-core/store"
107
9
	"launchpad.net/lpad"
108
10
	stdlog "log"
109
11
	"os"
110
12
	"path/filepath"
111
13
)
112
14
113
15
func main() {
114
16
	log.Target = stdlog.New(os.Stdout, "", stdlog.LstdFlags)
115
17
	err := load()
116
18
	if err != nil {
117
19
		fmt.Fprintf(os.Stderr, "%v\n", err)
118
20
		os.Exit(1)
119
21
	}
120
22
}
121
23
122
24
type config struct {
123
25
	MongoURL string `yaml:"mongo-url"`
124
26
}
125
27
126
28
func readConfig(path string, conf interface{}) error {
127
29
	f, err := os.Open(path)
128
30
	if err != nil {
129
31
		return fmt.Errorf("opening config file: %v", err)
130
32
	}
131
33
	data, err := ioutil.ReadAll(f)
132
34
	f.Close()
133
35
	if err != nil {
134
36
		return fmt.Errorf("reading config file: %v", err)
135
37
	}
136
38
	err = goyaml.Unmarshal(data, conf)
137
39
	if err != nil {
138
40
		return fmt.Errorf("processing config file: %v", err)
139
41
	}
140
42
	return nil
141
43
}
142
44
143
45
func load() error {
144
46
	var confPath string
145
47
	if len(os.Args) == 2 {
146
48
		if _, err := os.Stat(os.Args[1]); err == nil {
147
49
			confPath = os.Args[1]
148
50
		}
149
51
	}
150
52
	if confPath == "" {
151
53
		return fmt.Errorf("usage: %s <config path>", filepath.Base(os.Args[0]))
152
54
	}
153
55
	var conf config
154
56
	err := readConfig(confPath, &conf)
155
57
	if err != nil {
156
58
		return err
157
59
	}
158
60
	if conf.MongoURL == "" {
159
61
		return fmt.Errorf("missing mongo-url in config file")
160
62
	}
161
63
	s, err := store.Open(conf.MongoURL)
162
64
	if err != nil {
163
65
		return err
164
66
	}
165
67
	defer s.Close()
166
68
	err = store.PublishCharmsDistro(s, lpad.Production)
167
69
	if _, ok := err.(store.PublishBranchErrors); ok {
168
70
		// Ignore branch errors since they're commonplace here.
169
71
		// They're logged, though.
170
72
		return nil
171
73
	}
172
74
	return err
173
75
}
174
76
0
175
=== removed directory 'store'
176
=== removed file 'store/branch.go'
177
--- store/branch.go	2012-07-24 09:36:51 +0000
178
+++ store/branch.go	1970-01-01 00:00:00 +0000
179
@@ -1,152 +0,0 @@
180
1
package store
181
2
182
3
import (
183
4
	"bytes"
184
5
	"fmt"
185
6
	"io/ioutil"
186
7
	"launchpad.net/juju-core/charm"
187
8
	"os"
188
9
	"os/exec"
189
10
	"path/filepath"
190
11
	"strings"
191
12
)
192
13
193
14
// PublishBazaarBranch checks out the Bazaar branch from burl and
194
15
// publishes its latest revision at urls in the given store.
195
16
// The digest parameter must be the most recent known Bazaar
196
17
// revision id for the branch tip. If publishing this specific digest
197
18
// for these URLs has been attempted already, the publishing
198
19
// procedure may abort early. The published digest is the Bazaar
199
20
// revision id of the checked out branch's tip, though, which may
200
21
// differ from the digest parameter.
201
22
func PublishBazaarBranch(store *Store, urls []*charm.URL, burl string, digest string) error {
202
23
203
24
	// Prevent other publishers from updating these specific URLs
204
25
	// concurrently.
205
26
	lock, err := store.LockUpdates(urls)
206
27
	if err != nil {
207
28
		return err
208
29
	}
209
30
	defer lock.Unlock()
210
31
211
32
	var branchDir string
212
33
NewTip:
213
34
	// Prepare the charm publisher. This will compute the revision
214
35
	// to be assigned to the charm, and it will also fail if the
215
36
	// operation is unnecessary because charms are up-to-date.
216
37
	pub, err := store.CharmPublisher(urls, digest)
217
38
	if err != nil {
218
39
		return err
219
40
	}
220
41
221
42
	// Figure if publishing this charm was already attempted before and
222
43
	// failed. We won't try again endlessly if so. In the future we may
223
44
	// retry automatically in certain circumstances.
224
45
	event, err := store.CharmEvent(urls[0], digest)
225
46
	if err == nil && event.Kind != EventPublished {
226
47
		return fmt.Errorf("charm publishing previously failed: %s", strings.Join(event.Errors, "; "))
227
48
	} else if err != nil && err != ErrNotFound {
228
49
		return err
229
50
	}
230
51
231
52
	if branchDir == "" {
232
53
		// Retrieve the branch with a lightweight checkout, so that it
233
54
		// builds a working tree as cheaply as possible. History
234
55
		// doesn't matter here.
235
56
		tempDir, err := ioutil.TempDir("", "publish-branch-")
236
57
		if err != nil {
237
58
			return err
238
59
		}
239
60
		defer os.RemoveAll(tempDir)
240
61
		branchDir = filepath.Join(tempDir, "branch")
241
62
		output, err := exec.Command("bzr", "checkout", "--lightweight", burl, branchDir).CombinedOutput()
242
63
		if err != nil {
243
64
			return outputErr(output, err)
244
65
		}
245
66
246
67
		// Pick actual digest from tip. Publishing the real tip
247
68
		// revision rather than the revision for the digest provided is
248
69
		// strictly necessary to prevent a race condition. If the
249
70
		// provided digest was published instead, there's a chance
250
71
		// another publisher concurrently running could have found a
251
72
		// newer revision and published that first, and the digest
252
73
		// parameter provided is in fact an old version that would
253
74
		// overwrite the new version.
254
75
		tipDigest, err := bzrRevisionId(branchDir)
255
76
		if err != nil {
256
77
			return err
257
78
		}
258
79
		if tipDigest != digest {
259
80
			digest = tipDigest
260
81
			goto NewTip
261
82
		}
262
83
	}
263
84
264
85
	ch, err := charm.ReadDir(branchDir)
265
86
	if err == nil {
266
87
		// Hand over the charm to the store for bundling and
267
88
		// streaming its content into the database.
268
89
		err = pub.Publish(ch)
269
90
		if err == ErrUpdateConflict {
270
91
			// A conflict may happen in edge cases if the whole
271
92
			// locking mechanism fails due to an expiration event,
272
93
			// and then the expired concurrent publisher revives
273
94
			// for whatever reason and attempts to finish
274
95
			// publishing. The state of the system is still
275
96
			// consistent in that case, and the error isn't logged
276
97
			// since the revision was properly published before.
277
98
			return err
278
99
		}
279
100
	}
280
101
281
102
	// Publishing is done. Log failure or error.
282
103
	event = &CharmEvent{
283
104
		URLs:   urls,
284
105
		Digest: digest,
285
106
	}
286
107
	if err == nil {
287
108
		event.Kind = EventPublished
288
109
		event.Revision = pub.Revision()
289
110
	} else {
290
111
		event.Kind = EventPublishError
291
112
		event.Errors = []string{err.Error()}
292
113
	}
293
114
	if logerr := store.LogCharmEvent(event); logerr != nil {
294
115
		if err == nil {
295
116
			err = logerr
296
117
		} else {
297
118
			err = fmt.Errorf("%v; %v", err, logerr)
298
119
		}
299
120
	}
300
121
	return err
301
122
}
302
123
303
124
// bzrRevisionId returns the Bazaar revision id for the branch in branchDir.
304
125
func bzrRevisionId(branchDir string) (string, error) {
305
126
	cmd := exec.Command("bzr", "revision-info")
306
127
	cmd.Dir = branchDir
307
128
	stderr := &bytes.Buffer{}
308
129
	cmd.Stderr = stderr
309
130
	output, err := cmd.Output()
310
131
	if err != nil {
311
132
		output = append(output, '\n')
312
133
		output = append(output, stderr.Bytes()...)
313
134
		return "", outputErr(output, err)
314
135
	}
315
136
	pair := bytes.Fields(output)
316
137
	if len(pair) != 2 {
317
138
		output = append(output, '\n')
318
139
		output = append(output, stderr.Bytes()...)
319
140
		return "", fmt.Errorf(`invalid output from "bzr revision-info": %s`, output)
320
141
	}
321
142
	return string(pair[1]), nil
322
143
}
323
144
324
145
// outputErr returns an error that assembles some command's output and its
325
146
// error, if both output and err are set, and returns only err if output is nil.
326
147
func outputErr(output []byte, err error) error {
327
148
	if len(output) > 0 {
328
149
		return fmt.Errorf("%v\n%s", err, output)
329
150
	}
330
151
	return err
331
152
}
332
153
0
333
=== removed file 'store/branch_test.go'
334
--- store/branch_test.go	2012-10-09 01:19:01 +0000
335
+++ store/branch_test.go	1970-01-01 00:00:00 +0000
336
@@ -1,238 +0,0 @@
337
1
package store_test
338
2
339
3
import (
340
4
	"bytes"
341
5
	"fmt"
342
6
	"io/ioutil"
343
7
	. "launchpad.net/gocheck"
344
8
	"launchpad.net/juju-core/charm"
345
9
	"launchpad.net/juju-core/store"
346
10
	"launchpad.net/juju-core/testing"
347
11
	"os"
348
12
	"os/exec"
349
13
	"path/filepath"
350
14
	"strings"
351
15
	"time"
352
16
)
353
17
354
18
func (s *StoreSuite) dummyBranch(c *C, suffix string) bzrDir {
355
19
	tmpDir := c.MkDir()
356
20
	if suffix != "" {
357
21
		tmpDir = filepath.Join(tmpDir, suffix)
358
22
		err := os.MkdirAll(tmpDir, 0755)
359
23
		c.Assert(err, IsNil)
360
24
	}
361
25
	branch := bzrDir(tmpDir)
362
26
	branch.init()
363
27
364
28
	copyCharmDir(branch.path(), testing.Charms.Dir("series", "dummy"))
365
29
	branch.add()
366
30
	branch.commit("Imported charm.")
367
31
	return branch
368
32
}
369
33
370
34
var urls = []*charm.URL{
371
35
	charm.MustParseURL("cs:~joe/oneiric/dummy"),
372
36
	charm.MustParseURL("cs:oneiric/dummy"),
373
37
}
374
38
375
39
type fakePlugin struct {
376
40
	oldEnv string
377
41
}
378
42
379
43
func (p *fakePlugin) install(dir string, content string) {
380
44
	p.oldEnv = os.Getenv("BZR_PLUGINS_AT")
381
45
	err := ioutil.WriteFile(filepath.Join(dir, "__init__.py"), []byte(content), 0644)
382
46
	if err != nil {
383
47
		panic(err)
384
48
	}
385
49
	os.Setenv("BZR_PLUGINS_AT", "fakePlugin@"+dir)
386
50
}
387
51
388
52
func (p *fakePlugin) uninstall() {
389
53
	os.Setenv("BZR_PLUGINS_AT", p.oldEnv)
390
54
}
391
55
392
56
func (s *StoreSuite) TestPublish(c *C) {
393
57
	branch := s.dummyBranch(c, "")
394
58
395
59
	// Ensure that the streams are parsed separately by inserting
396
60
	// garbage on stderr. The wanted information is still there.
397
61
	plugin := fakePlugin{}
398
62
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n")`)
399
63
	defer plugin.uninstall()
400
64
401
65
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
402
66
	c.Assert(err, IsNil)
403
67
404
68
	for _, url := range urls {
405
69
		info, rc, err := s.store.OpenCharm(url)
406
70
		c.Assert(err, IsNil)
407
71
		defer rc.Close()
408
72
		c.Assert(info.Revision(), Equals, 0)
409
73
		c.Assert(info.Meta().Name, Equals, "dummy")
410
74
411
75
		data, err := ioutil.ReadAll(rc)
412
76
		c.Assert(err, IsNil)
413
77
414
78
		bundle, err := charm.ReadBundleBytes(data)
415
79
		c.Assert(err, IsNil)
416
80
		c.Assert(bundle.Revision(), Equals, 0)
417
81
		c.Assert(bundle.Meta().Name, Equals, "dummy")
418
82
	}
419
83
420
84
	// Attempt to publish the same content again while providing the wrong
421
85
	// tip revision. It must pick the real revision from the branch and
422
86
	// note this was previously published.
423
87
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
424
88
	c.Assert(err, Equals, store.ErrRedundantUpdate)
425
89
426
90
	// Bump the content revision and lie again about the known tip revision.
427
91
	// This time, though, pretend it's the same as the real branch revision
428
92
	// previously published. It must error and not publish the new revision
429
93
	// because it will use the revision provided as a parameter to check if
430
94
	// publishing was attempted before. This is the mechanism that enables
431
95
	// stopping fast without having to download every single branch. Real
432
96
	// revision is picked in the next scan.
433
97
	digest1 := branch.digest()
434
98
	branch.change()
435
99
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), digest1)
436
100
	c.Assert(err, Equals, store.ErrRedundantUpdate)
437
101
438
102
	// Now allow it to publish the new content by providing an unseen revision.
439
103
	err = store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
440
104
	c.Assert(err, IsNil)
441
105
	digest2 := branch.digest()
442
106
443
107
	info, err := s.store.CharmInfo(urls[0])
444
108
	c.Assert(err, IsNil)
445
109
	c.Assert(info.Revision(), Equals, 1)
446
110
	c.Assert(info.Meta().Name, Equals, "dummy")
447
111
448
112
	// There are two events published, for each of the successful attempts.
449
113
	// The failures are ignored given that they are artifacts of the
450
114
	// publishing mechanism rather than actual problems.
451
115
	_, err = s.store.CharmEvent(urls[0], "wrong-rev")
452
116
	c.Assert(err, Equals, store.ErrNotFound)
453
117
	for i, digest := range []string{digest1, digest2} {
454
118
		event, err := s.store.CharmEvent(urls[0], digest)
455
119
		c.Assert(err, IsNil)
456
120
		c.Assert(event.Kind, Equals, store.EventPublished)
457
121
		c.Assert(event.Revision, Equals, i)
458
122
		c.Assert(event.Errors, IsNil)
459
123
		c.Assert(event.Warnings, IsNil)
460
124
	}
461
125
}
462
126
463
127
func (s *StoreSuite) TestPublishErrorFromBzr(c *C) {
464
128
	branch := s.dummyBranch(c, "")
465
129
466
130
	// In TestPublish we ensure that the streams are parsed
467
131
	// separately by inserting garbage on stderr. Now make
468
132
	// sure that stderr isn't simply trashed, as we want to
469
133
	// know about what a real error tells us.
470
134
	plugin := fakePlugin{}
471
135
	plugin.install(c.MkDir(), `import sys; sys.stderr.write("STDERR STUFF FROM TEST\n"); sys.exit(1)`)
472
136
	defer plugin.uninstall()
473
137
474
138
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
475
139
	c.Assert(err, ErrorMatches, "(?s).*STDERR STUFF.*")
476
140
}
477
141
478
142
func (s *StoreSuite) TestPublishErrorInCharm(c *C) {
479
143
	branch := s.dummyBranch(c, "")
480
144
481
145
	// Corrupt the charm.
482
146
	branch.remove("metadata.yaml")
483
147
	branch.commit("Removed metadata.yaml.")
484
148
485
149
	// Attempt to publish the erroneous content.
486
150
	err := store.PublishBazaarBranch(s.store, urls, branch.path(), "wrong-rev")
487
151
	c.Assert(err, ErrorMatches, ".*/metadata.yaml: no such file or directory")
488
152
489
153
	// The event should be logged as well, since this was an error in the charm
490
154
	// that won't go away and must be communicated to the author.
491
155
	event, err := s.store.CharmEvent(urls[0], branch.digest())
492
156
	c.Assert(err, IsNil)
493
157
	c.Assert(event.Kind, Equals, store.EventPublishError)
494
158
	c.Assert(event.Revision, Equals, 0)
495
159
	c.Assert(event.Errors, NotNil)
496
160
	c.Assert(event.Errors[0], Matches, ".*/metadata.yaml: no such file or directory")
497
161
	c.Assert(event.Warnings, IsNil)
498
162
}
499
163
500
164
type bzrDir string
501
165
502
166
func (dir bzrDir) path(args ...string) string {
503
167
	return filepath.Join(append([]string{string(dir)}, args...)...)
504
168
}
505
169
506
170
func (dir bzrDir) run(args ...string) []byte {
507
171
	cmd := exec.Command("bzr", args...)
508
172
	oldemail := os.Getenv("EMAIL")
509
173
	defer os.Setenv("EMAIL", oldemail)
510
174
	// bzr will complain if bzr whoami has not been run previously,
511
175
	// avoid this by passing $EMAIL into the environment.
512
176
	os.Setenv("EMAIL", "nobody@example.com")
513
177
	cmd.Dir = string(dir)
514
178
	output, err := cmd.Output()
515
179
	if err != nil {
516
180
		panic(fmt.Sprintf("command failed: bzr %s\n%s", strings.Join(args, " "), output))
517
181
	}
518
182
	return output
519
183
}
520
184
521
185
func (dir bzrDir) init() {
522
186
	dir.run("init")
523
187
}
524
188
525
189
func (dir bzrDir) add(paths ...string) {
526
190
	dir.run(append([]string{"add"}, paths...)...)
527
191
}
528
192
529
193
func (dir bzrDir) remove(paths ...string) {
530
194
	dir.run(append([]string{"rm"}, paths...)...)
531
195
}
532
196
533
197
func (dir bzrDir) commit(msg string) {
534
198
	dir.run("commit", "-m", msg)
535
199
}
536
200
537
201
func (dir bzrDir) write(path string, data string) {
538
202
	err := ioutil.WriteFile(dir.path(path), []byte(data), 0644)
539
203
	if err != nil {
540
204
		panic(err)
541
205
	}
542
206
}
543
207
544
208
func (dir bzrDir) change() {
545
209
	t := time.Now().String()
546
210
	dir.write("timestamp", t)
547
211
	dir.add("timestamp")
548
212
	dir.commit("Revision bumped at " + t)
549
213
}
550
214
551
215
func (dir bzrDir) digest() string {
552
216
	output := dir.run("revision-info")
553
217
	f := bytes.Fields(output)
554
218
	if len(f) != 2 {
555
219
		panic("revision-info returned bad output: " + string(output))
556
220
	}
557
221
	return string(f[1])
558
222
}
559
223
560
224
func copyCharmDir(dst string, dir *charm.Dir) {
561
225
	var b bytes.Buffer
562
226
	err := dir.BundleTo(&b)
563
227
	if err != nil {
564
228
		panic(err)
565
229
	}
566
230
	bundle, err := charm.ReadBundleBytes(b.Bytes())
567
231
	if err != nil {
568
232
		panic(err)
569
233
	}
570
234
	err = bundle.ExpandTo(dst)
571
235
	if err != nil {
572
236
		panic(err)
573
237
	}
574
238
}
575
239
0
576
=== removed file 'store/lpad.go'
577
--- store/lpad.go	2012-06-21 20:40:39 +0000
578
+++ store/lpad.go	1970-01-01 00:00:00 +0000
579
@@ -1,113 +0,0 @@
580
1
package store
581
2
582
3
import (
583
4
	"fmt"
584
5
	"launchpad.net/juju-core/charm"
585
6
	"launchpad.net/juju-core/log"
586
7
	"launchpad.net/lpad"
587
8
	"strings"
588
9
	"time"
589
10
)
590
11
591
12
type PublishBranchError struct {
592
13
	URL string
593
14
	Err error
594
15
}
595
16
596
17
type PublishBranchErrors []PublishBranchError
597
18
598
19
func (errs PublishBranchErrors) Error() string {
599
20
	return fmt.Sprintf("%d branch(es) failed to be published", len(errs))
600
21
}
601
22
602
23
// PublishCharmsDistro publishes all branch tips found in
603
24
// the /charms distribution in Launchpad onto store under
604
25
// the "cs:" scheme.
605
26
// apiBase specifies the Launchpad base API URL, such
606
27
// as lpad.Production or lpad.Staging.
607
28
// Errors found while processing one or more branches are
608
29
// all returned as a PublishBranchErrors value.
609
30
func PublishCharmsDistro(store *Store, apiBase lpad.APIBase) error {
610
31
	oauth := &lpad.OAuth{Anonymous: true, Consumer: "juju"}
611
32
	root, err := lpad.Login(apiBase, oauth)
612
33
	if err != nil {
613
34
		return err
614
35
	}
615
36
	distro, err := root.Distro("charms")
616
37
	if err != nil {
617
38
		return err
618
39
	}
619
40
	tips, err := distro.BranchTips(time.Time{})
620
41
	if err != nil {
621
42
		return err
622
43
	}
623
44
624
45
	var errs PublishBranchErrors
625
46
	for _, tip := range tips {
626
47
		if !strings.HasSuffix(tip.UniqueName, "/trunk") {
627
48
			continue
628
49
		}
629
50
		burl, curl, err := uniqueNameURLs(tip.UniqueName)
630
51
		if err != nil {
631
52
			errs = append(errs, PublishBranchError{tip.UniqueName, err})
632
53
			log.Printf("error: %v\n", err)
633
54
			continue
634
55
		}
635
56
		log.Printf("----- %s\n", burl)
636
57
		if tip.Revision == "" {
637
58
			errs = append(errs, PublishBranchError{burl, fmt.Errorf("branch has no revisions")})
638
59
			log.Printf("error: branch has no revisions\n")
639
60
			continue
640
61
		}
641
62
		// Charm is published in the personal URL and in any explicitly
642
63
		// assigned official series.
643
64
		urls := []*charm.URL{curl}
644
65
		schema, name := curl.Schema, curl.Name
645
66
		for _, series := range tip.OfficialSeries {
646
67
			curl = &charm.URL{Schema: schema, Name: name, Series: series, Revision: -1}
647
68
			curl.Series = series
648
69
			curl.User = ""
649
70
			urls = append(urls, curl)
650
71
		}
651
72
652
73
		err = PublishBazaarBranch(store, urls, burl, tip.Revision)
653
74
		if err == ErrRedundantUpdate {
654
75
			continue
655
76
		}
656
77
		if err != nil {
657
78
			errs = append(errs, PublishBranchError{burl, err})
658
79
			log.Printf("error: %v\n", err)
659
80
		}
660
81
	}
661
82
	if errs != nil {
662
83
		return errs
663
84
	}
664
85
	return nil
665
86
}
666
87
667
88
// uniqueNameURLs returns the branch URL and the charm URL for the
668
89
// provided Launchpad branch unique name. The unique name must be
669
90
// in the form:
670
91
//
671
92
//     ~<user>/charms/<series>/<charm name>/trunk
672
93
//
673
94
// For testing purposes, if name has a prefix preceding a string in
674
95
// this format, the prefix is stripped out for computing the charm
675
96
// URL, and the unique name is returned unchanged as the branch URL.
676
97
func uniqueNameURLs(name string) (burl string, curl *charm.URL, err error) {
677
98
	u := strings.Split(name, "/")
678
99
	if len(u) > 5 {
679
100
		u = u[len(u)-5:]
680
101
		burl = name
681
102
	} else {
682
103
		burl = "lp:" + name
683
104
	}
684
105
	if len(u) < 5 || u[1] != "charms" || u[4] != "trunk" || len(u[0]) == 0 || u[0][0] != '~' {
685
106
		return "", nil, fmt.Errorf("unwanted branch name: %s", name)
686
107
	}
687
108
	curl, err = charm.ParseURL(fmt.Sprintf("cs:%s/%s/%s", u[0], u[2], u[3]))
688
109
	if err != nil {
689
110
		return "", nil, err
690
111
	}
691
112
	return burl, curl, nil
692
113
}
693
114
0
694
=== removed file 'store/lpad_test.go'
695
--- store/lpad_test.go	2012-08-18 22:48:02 +0000
696
+++ store/lpad_test.go	1970-01-01 00:00:00 +0000
697
@@ -1,68 +0,0 @@
698
1
package store_test
699
2
700
3
import (
701
4
	"fmt"
702
5
	. "launchpad.net/gocheck"
703
6
	"launchpad.net/juju-core/charm"
704
7
	"launchpad.net/juju-core/store"
705
8
	"launchpad.net/juju-core/testing"
706
9
	"launchpad.net/lpad"
707
10
)
708
11
709
12
var jsonType = map[string]string{
710
13
	"Content-Type": "application/json",
711
14
}
712
15
713
16
func (s *StoreSuite) TestPublishCharmDistro(c *C) {
714
17
	branch := s.dummyBranch(c, "~joe/charms/oneiric/dummy/trunk")
715
18
716
19
	// The Distro call will look for bare /charms, first.
717
20
	testing.Server.Response(200, jsonType, []byte("{}"))
718
21
719
22
	// And then it picks up the tips.
720
23
	data := fmt.Sprintf(`[`+
721
24
		`["file://%s", "rev1", ["oneiric", "precise"]],`+
722
25
		`["file://%s", "%s", []],`+
723
26
		`["file:///non-existent/~jeff/charms/precise/bad/trunk", "rev2", []],`+
724
27
		`["file:///non-existent/~jeff/charms/precise/bad/skip-me", "rev3", []]`+
725
28
		`]`,
726
29
		branch.path(), branch.path(), branch.digest())
727
30
	testing.Server.Response(200, jsonType, []byte(data))
728
31
729
32
	apiBase := lpad.APIBase(testing.Server.URL)
730
33
	err := store.PublishCharmsDistro(s.store, apiBase)
731
34
732
35
	// Should have a single failure from the trunk branch that doesn't
733
36
	// exist. The redundant update with the known digest should be
734
37
	// ignored, and skip-me isn't a supported branch name so it's
735
38
	// ignored as well.
736
39
	c.Assert(err, ErrorMatches, `1 branch\(es\) failed to be published`)
737
40
	berr := err.(store.PublishBranchErrors)[0]
738
41
	c.Assert(berr.URL, Equals, "file:///non-existent/~jeff/charms/precise/bad/trunk")
739
42
	c.Assert(berr.Err, ErrorMatches, "(?s).*bzr: ERROR: Not a branch.*")
740
43
741
44
	for _, url := range []string{"cs:oneiric/dummy", "cs:precise/dummy-0", "cs:~joe/oneiric/dummy-0"} {
742
45
		dummy, err := s.store.CharmInfo(charm.MustParseURL(url))
743
46
		c.Assert(err, IsNil)
744
47
		c.Assert(dummy.Meta().Name, Equals, "dummy")
745
48
	}
746
49
747
50
	// The known digest should have been ignored, so revision is still at 0.
748
51
	_, err = s.store.CharmInfo(charm.MustParseURL("cs:~joe/oneiric/dummy-1"))
749
52
	c.Assert(err, Equals, store.ErrNotFound)
750
53
751
54
	// bare /charms lookup
752
55
	req := testing.Server.WaitRequest()
753
56
	c.Assert(req.Method, Equals, "GET")
754
57
	c.Assert(req.URL.Path, Equals, "/charms")
755
58
756
59
	// tips request
757
60
	req = testing.Server.WaitRequest()
758
61
	c.Assert(req.Method, Equals, "GET")
759
62
	c.Assert(req.URL.Path, Equals, "/charms")
760
63
	c.Assert(req.Form["ws.op"], DeepEquals, []string{"getBranchTips"})
761
64
	c.Assert(req.Form["since"], IsNil)
762
65
763
66
	// Request must be signed by juju.
764
67
	c.Assert(req.Header.Get("Authorization"), Matches, `.*oauth_consumer_key="juju".*`)
765
68
}
766
69
0
767
=== removed file 'store/mgo_test.go'
768
--- store/mgo_test.go	2012-10-30 11:45:32 +0000
769
+++ store/mgo_test.go	1970-01-01 00:00:00 +0000
770
@@ -1,95 +0,0 @@
771
1
package store_test
772
2
773
3
import (
774
4
	"bytes"
775
5
	"labix.org/v2/mgo"
776
6
	. "launchpad.net/gocheck"
777
7
	"os/exec"
778
8
	"time"
779
9
)
780
10
781
11
// ----------------------------------------------------------------------------
782
12
// The mgo test suite
783
13
784
14
type MgoSuite struct {
785
15
	Addr    string
786
16
	Session *mgo.Session
787
17
	output  bytes.Buffer
788
18
	server  *exec.Cmd
789
19
}
790
20
791
21
func (s *MgoSuite) SetUpSuite(c *C) {
792
22
	mgo.SetDebug(true)
793
23
	mgo.SetStats(true)
794
24
	dbdir := c.MkDir()
795
25
	args := []string{
796
26
		"--dbpath", dbdir,
797
27
		"--bind_ip", "127.0.0.1",
798
28
		"--port", "50017",
799
29
		"--nssize", "1",
800
30
		"--noprealloc",
801
31
		"--smallfiles",
802
32
		"--nojournal",
803
33
	}
804
34
	s.server = exec.Command("mongod", args...)
805
35
	s.server.Stdout = &s.output
806
36
	s.server.Stderr = &s.output
807
37
	err := s.server.Start()
808
38
	c.Assert(err, IsNil)
809
39
}
810
40
811
41
func (s *MgoSuite) TearDownSuite(c *C) {
812
42
	s.server.Process.Kill()
813
43
	s.server.Process.Wait()
814
44
}
815
45
816
46
func (s *MgoSuite) SetUpTest(c *C) {
817
47
	err := DropAll("localhost:50017")
818
48
	c.Assert(err, IsNil)
819
49
	mgo.SetLogger(c)
820
50
	mgo.ResetStats()
821
51
	s.Addr = "127.0.0.1:50017"
822
52
	s.Session, err = mgo.Dial(s.Addr)
823
53
	c.Assert(err, IsNil)
824
54
}
825
55
826
56
func (s *MgoSuite) TearDownTest(c *C) {
827
57
	if s.Session != nil {
828
58
		s.Session.Close()
829
59
	}
830
60
	for i := 0; ; i++ {
831
61
		stats := mgo.GetStats()
832
62
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
833
63
			break
834
64
		}
835
65
		if i == 20 {
836
66
			c.Fatal("Test left sockets in a dirty state")
837
67
		}
838
68
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
839
69
		time.Sleep(500 * time.Millisecond)
840
70
	}
841
71
}
842
72
843
73
func DropAll(mongourl string) (err error) {
844
74
	session, err := mgo.Dial(mongourl)
845
75
	if err != nil {
846
76
		return err
847
77
	}
848
78
	defer session.Close()
849
79
850
80
	names, err := session.DatabaseNames()
851
81
	if err != nil {
852
82
		return err
853
83
	}
854
84
	for _, name := range names {
855
85
		switch name {
856
86
		case "admin", "local", "config":
857
87
		default:
858
88
			err = session.DB(name).DropDatabase()
859
89
			if err != nil {
860
90
				return err
861
91
			}
862
92
		}
863
93
	}
864
94
	return nil
865
95
}
866
96
0
867
=== removed file 'store/server.go'
868
--- store/server.go	2012-10-11 14:52:21 +0000
869
+++ store/server.go	1970-01-01 00:00:00 +0000
870
@@ -1,191 +0,0 @@
871
1
package store
872
2
873
3
import (
874
4
	"encoding/json"
875
5
	"io"
876
6
	"launchpad.net/juju-core/charm"
877
7
	"launchpad.net/juju-core/log"
878
8
	"net/http"
879
9
	"strconv"
880
10
	"strings"
881
11
)
882
12
883
13
// Server is an http.Handler that serves the HTTP API of juju
884
14
// so that juju clients can retrieve published charms.
885
15
type Server struct {
886
16
	store *Store
887
17
	mux   *http.ServeMux
888
18
}
889
19
890
20
// New returns a new *Server using store.
891
21
func NewServer(store *Store) (*Server, error) {
892
22
	s := &Server{
893
23
		store: store,
894
24
		mux:   http.NewServeMux(),
895
25
	}
896
26
	s.mux.HandleFunc("/charm-info", func(w http.ResponseWriter, r *http.Request) {
897
27
		s.serveInfo(w, r)
898
28
	})
899
29
	s.mux.HandleFunc("/charm/", func(w http.ResponseWriter, r *http.Request) {
900
30
		s.serveCharm(w, r)
901
31
	})
902
32
	s.mux.HandleFunc("/stats/counter/", func(w http.ResponseWriter, r *http.Request) {
903
33
		s.serveStats(w, r)
904
34
	})
905
35
906
36
	// This is just a validation key to allow blitz.io to run
907
37
	// performance tests against the site.
908
38
	s.mux.HandleFunc("/mu-35700a31-6bf320ca-a800b670-05f845ee", func(w http.ResponseWriter, r *http.Request) {
909
39
		s.serveBlitzKey(w, r)
910
40
	})
911
41
	return s, nil
912
42
}
913
43
914
44
// ServeHTTP serves an http request.
915
45
// This method turns *Server into an http.Handler.
916
46
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
917
47
	if r.URL.Path == "/" {
918
48
		http.Redirect(w, r, "https://juju.ubuntu.com", http.StatusSeeOther)
919
49
		return
920
50
	}
921
51
	s.mux.ServeHTTP(w, r)
922
52
}
923
53
924
54
func statsEnabled(req *http.Request) bool {
925
55
	// It's fine to parse the form more than once, and it avoids
926
56
	// bugs from not parsing it.
927
57
	req.ParseForm()
928
58
	return req.Form.Get("stats") != "0"
929
59
}
930
60
931
61
func charmStatsKey(curl *charm.URL, kind string) []string {
932
62
	if curl.User == "" {
933
63
		return []string{kind, curl.Series, curl.Name}
934
64
	}
935
65
	return []string{kind, curl.Series, curl.Name, curl.User}
936
66
}
937
67
938
68
func (s *Server) serveInfo(w http.ResponseWriter, r *http.Request) {
939
69
	if r.URL.Path != "/charm-info" {
940
70
		w.WriteHeader(http.StatusNotFound)
941
71
		return
942
72
	}
943
73
	r.ParseForm()
944
74
	response := map[string]*charm.InfoResponse{}
945
75
	for _, url := range r.Form["charms"] {
946
76
		c := &charm.InfoResponse{}
947
77
		response[url] = c
948
78
		curl, err := charm.ParseURL(url)
949
79
		var info *CharmInfo
950
80
		if err == nil {
951
81
			info, err = s.store.CharmInfo(curl)
952
82
		}
953
83
		var skey []string
954
84
		if err == nil {
955
85
			skey = charmStatsKey(curl, "charm-info")
956
86
			c.Sha256 = info.BundleSha256()
957
87
			c.Revision = info.Revision()
958
88
			c.Digest = info.Digest()
959
89
		} else {
960
90
			if err == ErrNotFound {
961
91
				skey = charmStatsKey(curl, "charm-missing")
962
92
			}
963
93
			c.Errors = append(c.Errors, err.Error())
964
94
		}
965
95
		if skey != nil && statsEnabled(r) {
966
96
			go s.store.IncCounter(skey)
967
97
		}
968
98
	}
969
99
	data, err := json.Marshal(response)
970
100
	if err == nil {
971
101
		w.Header().Set("Content-Type", "application/json")
972
102
		_, err = w.Write(data)
973
103
	}
974
104
	if err != nil {
975
105
		log.Printf("store: cannot write content: %v", err)
976
106
		w.WriteHeader(http.StatusInternalServerError)
977
107
		return
978
108
	}
979
109
}
980
110
981
111
func (s *Server) serveCharm(w http.ResponseWriter, r *http.Request) {
982
112
	if !strings.HasPrefix(r.URL.Path, "/charm/") {
983
113
		panic("serveCharm: bad url")
984
114
	}
985
115
	curl, err := charm.ParseURL("cs:" + r.URL.Path[len("/charm/"):])
986
116
	if err != nil {
987
117
		w.WriteHeader(http.StatusNotFound)
988
118
		return
989
119
	}
990
120
	info, rc, err := s.store.OpenCharm(curl)
991
121
	if err == ErrNotFound {
992
122
		w.WriteHeader(http.StatusNotFound)
993
123
		return
994
124
	}
995
125
	if err != nil {
996
126
		w.WriteHeader(http.StatusInternalServerError)
997
127
		log.Printf("store: cannot open charm %q: %v", curl, err)
998
128
		return
999
129
	}
1000
130
	if statsEnabled(r) {
1001
131
		go s.store.IncCounter(charmStatsKey(curl, "charm-bundle"))
1002
132
	}
1003
133
	defer rc.Close()
1004
134
	w.Header().Set("Connection", "close") // No keep-alive for now.
1005
135
	w.Header().Set("Content-Type", "application/octet-stream")
1006
136
	w.Header().Set("Content-Length", strconv.FormatInt(info.BundleSize(), 10))
1007
137
	_, err = io.Copy(w, rc)
1008
138
	if err != nil {
1009
139
		log.Printf("store: failed to stream charm %q: %v", curl, err)
1010
140
	}
1011
141
}
1012
142
1013
143
func (s *Server) serveStats(w http.ResponseWriter, r *http.Request) {
1014
144
	// TODO: Adopt a smarter mux that simplifies this logic.
1015
145
	const dir = "/stats/counter/"
1016
146
	if !strings.HasPrefix(r.URL.Path, dir) {
1017
147
		panic("bad url")
1018
148
	}
1019
149
	base := r.URL.Path[len(dir):]
1020
150
	if strings.Index(base, "/") > 0 {
1021
151
		w.WriteHeader(http.StatusNotFound)
1022
152
		return
1023
153
	}
1024
154
	if base == "" {
1025
155
		w.WriteHeader(http.StatusForbidden)
1026
156
		return
1027
157
	}
1028
158
	key := strings.Split(base, ":")
1029
159
	prefix := false
1030
160
	if key[len(key)-1] == "*" {
1031
161
		prefix = true
1032
162
		key = key[:len(key)-1]
1033
163
		if len(key) == 0 {
1034
164
			// No point in counting something unknown.
1035
165
			w.WriteHeader(http.StatusForbidden)
1036
166
			return
1037
167
		}
1038
168
	}
1039
169
	r.ParseForm()
1040
170
	sum, err := s.store.SumCounter(key, prefix)
1041
171
	if err != nil {
1042
172
		log.Printf("store: cannot sum counter: %v", err)
1043
173
		w.WriteHeader(http.StatusInternalServerError)
1044
174
		return
1045
175
	}
1046
176
	data := []byte(strconv.FormatInt(sum, 10))
1047
177
	w.Header().Set("Content-Type", "text/plain")
1048
178
	w.Header().Set("Content-Length", strconv.Itoa(len(data)))
1049
179
	_, err = w.Write(data)
1050
180
	if err != nil {
1051
181
		log.Printf("store: cannot write content: %v", err)
1052
182
		w.WriteHeader(http.StatusInternalServerError)
1053
183
	}
1054
184
}
1055
185
1056
186
func (s *Server) serveBlitzKey(w http.ResponseWriter, r *http.Request) {
1057
187
	w.Header().Set("Connection", "close")
1058
188
	w.Header().Set("Content-Type", "text/plain")
1059
189
	w.Header().Set("Content-Length", "2")
1060
190
	w.Write([]byte("42"))
1061
191
}
1062
192
0
1063
=== removed file 'store/server_test.go'
1064
--- store/server_test.go	2012-09-05 21:08:26 +0000
1065
+++ store/server_test.go	1970-01-01 00:00:00 +0000
1066
@@ -1,209 +0,0 @@
1067
1
package store_test
1068
2
1069
3
import (
1070
4
	"encoding/json"
1071
5
	"io/ioutil"
1072
6
	. "launchpad.net/gocheck"
1073
7
	"launchpad.net/juju-core/charm"
1074
8
	"launchpad.net/juju-core/store"
1075
9
	"net/http"
1076
10
	"net/http/httptest"
1077
11
	"net/url"
1078
12
	"strconv"
1079
13
	"time"
1080
14
)
1081
15
1082
16
func (s *StoreSuite) prepareServer(c *C) (*store.Server, *charm.URL) {
1083
17
	curl := charm.MustParseURL("cs:oneiric/wordpress")
1084
18
	pub, err := s.store.CharmPublisher([]*charm.URL{curl}, "some-digest")
1085
19
	c.Assert(err, IsNil)
1086
20
	err = pub.Publish(&FakeCharmDir{})
1087
21
	c.Assert(err, IsNil)
1088
22
1089
23
	server, err := store.NewServer(s.store)
1090
24
	c.Assert(err, IsNil)
1091
25
	return server, curl
1092
26
}
1093
27
1094
28
func (s *StoreSuite) TestServerCharmInfo(c *C) {
1095
29
	server, curl := s.prepareServer(c)
1096
30
	req, err := http.NewRequest("GET", "/charm-info", nil)
1097
31
	c.Assert(err, IsNil)
1098
32
1099
33
	var tests = []struct{ url, sha, digest, err string }{
1100
34
		{curl.String(), fakeRevZeroSha, "some-digest", ""},
1101
35
		{"cs:oneiric/non-existent", "", "", "entry not found"},
1102
36
		{"cs:bad", "", "", `charm URL without series: "cs:bad"`},
1103
37
	}
1104
38
1105
39
	for _, t := range tests {
1106
40
		req.Form = url.Values{"charms": []string{t.url}}
1107
41
		rec := httptest.NewRecorder()
1108
42
		server.ServeHTTP(rec, req)
1109
43
1110
44
		expected := make(map[string]interface{})
1111
45
		if t.sha != "" {
1112
46
			expected[t.url] = map[string]interface{}{
1113
47
				"revision": float64(0),
1114
48
				"sha256":   t.sha,
1115
49
				"digest":   t.digest,
1116
50
			}
1117
51
		} else {
1118
52
			expected[t.url] = map[string]interface{}{
1119
53
				"revision": float64(0),
1120
54
				"errors":   []interface{}{t.err},
1121
55
			}
1122
56
		}
1123
57
		obtained := map[string]interface{}{}
1124
58
		err = json.NewDecoder(rec.Body).Decode(&obtained)
1125
59
		c.Assert(err, IsNil)
1126
60
		c.Assert(obtained, DeepEquals, expected)
1127
61
		c.Assert(rec.Header().Get("Content-Type"), Equals, "application/json")
1128
62
	}
1129
63
1130
64
	s.checkCounterSum(c, []string{"charm-info", curl.Series, curl.Name}, false, 1)
1131
65
	s.checkCounterSum(c, []string{"charm-missing", "oneiric", "non-existent"}, false, 1)
1132
66
}
1133
67
1134
68
// checkCounterSum checks that statistics are properly collected.
1135
69
// It retries a few times as they are generally collected in background.
1136
70
func (s *StoreSuite) checkCounterSum(c *C, key []string, prefix bool, expected int64) {
1137
71
	var sum int64
1138
72
	var err error
1139
73
	for retry := 0; retry < 10; retry++ {
1140
74
		time.Sleep(1e8)
1141
75
		sum, err = s.store.SumCounter(key, prefix)
1142
76
		c.Assert(err, IsNil)
1143
77
		if sum == expected {
1144
78
			if expected == 0 && retry < 2 {
1145
79
				continue // Wait a bit to make sure.
1146
80
			}
1147
81
			return
1148
82
		}
1149
83
	}
1150
84
	c.Errorf("counter sum for %#v is %d, want %d", key, sum, expected)
1151
85
}
1152
86
1153
87
func (s *StoreSuite) TestCharmStreaming(c *C) {
1154
88
	server, curl := s.prepareServer(c)
1155
89
1156
90
	req, err := http.NewRequest("GET", "/charm/"+curl.String()[3:], nil)
1157
91
	c.Assert(err, IsNil)
1158
92
	rec := httptest.NewRecorder()
1159
93
	server.ServeHTTP(rec, req)
1160
94
1161
95
	data, err := ioutil.ReadAll(rec.Body)
1162
96
	c.Assert(string(data), Equals, "charm-revision-0")
1163
97
1164
98
	c.Assert(rec.Header().Get("Connection"), Equals, "close")
1165
99
	c.Assert(rec.Header().Get("Content-Type"), Equals, "application/octet-stream")
1166
100
	c.Assert(rec.Header().Get("Content-Length"), Equals, "16")
1167
101
1168
102
	// Check that it was accounted for in statistics.
1169
103
	s.checkCounterSum(c, []string{"charm-bundle", curl.Series, curl.Name}, false, 1)
1170
104
}
1171
105
1172
106
func (s *StoreSuite) TestDisableStats(c *C) {
1173
107
	server, curl := s.prepareServer(c)
1174
108
1175
109
	req, err := http.NewRequest("GET", "/charm-info", nil)
1176
110
	c.Assert(err, IsNil)
1177
111
	req.Form = url.Values{"charms": []string{curl.String()}, "stats": []string{"0"}}
1178
112
	rec := httptest.NewRecorder()
1179
113
	server.ServeHTTP(rec, req)
1180
114
	c.Assert(rec.Code, Equals, 200)
1181
115
1182
116
	req, err = http.NewRequest("GET", "/charm/"+curl.String()[3:], nil)
1183
117
	c.Assert(err, IsNil)
1184
118
	req.Form = url.Values{"stats": []string{"0"}}
1185
119
	rec = httptest.NewRecorder()
1186
120
	server.ServeHTTP(rec, req)
1187
121
	c.Assert(rec.Code, Equals, 200)
1188
122
1189
123
	// No statistics should have been collected given the use of stats=0.
1190
124
	for _, prefix := range []string{"charm-info", "charm-bundle", "charm-missing"} {
1191
125
		s.checkCounterSum(c, []string{prefix}, true, 0)
1192
126
	}
1193
127
}
1194
128
1195
129
func (s *StoreSuite) TestServerStatus(c *C) {
1196
130
	server, err := store.NewServer(s.store)
1197
131
	c.Assert(err, IsNil)
1198
132
	tests := []struct {
1199
133
		path string
1200
134
		code int
1201
135
	}{
1202
136
		{"/charm-info/any", 404},
1203
137
		{"/charm/bad-url", 404},
1204
138
		{"/charm/bad-series/wordpress", 404},
1205
139
		{"/stats/counter/", 403},
1206
140
		{"/stats/counter/*", 403},
1207
141
		{"/stats/counter/any/", 404},
1208
142
		{"/stats/", 404},
1209
143
		{"/stats/any", 404},
1210
144
	}
1211
145
	for _, test := range tests {
1212
146
		req, err := http.NewRequest("GET", test.path, nil)
1213
147
		c.Assert(err, IsNil)
1214
148
		rec := httptest.NewRecorder()
1215
149
		server.ServeHTTP(rec, req)
1216
150
		c.Assert(rec.Code, Equals, test.code, Commentf("Path: %s", test.path))
1217
151
	}
1218
152
}
1219
153
1220
154
func (s *StoreSuite) TestRootRedirect(c *C) {
1221
155
	server, err := store.NewServer(s.store)
1222
156
	c.Assert(err, IsNil)
1223
157
	req, err := http.NewRequest("GET", "/", nil)
1224
158
	c.Assert(err, IsNil)
1225
159
	rec := httptest.NewRecorder()
1226
160
	server.ServeHTTP(rec, req)
1227
161
	c.Assert(rec.Code, Equals, 303)
1228
162
	c.Assert(rec.Header().Get("Location"), Equals, "https://juju.ubuntu.com")
1229
163
}
1230
164
1231
165
func (s *StoreSuite) TestStatsCounter(c *C) {
1232
166
	for _, key := range [][]string{{"a", "b"}, {"a", "b"}, {"a"}} {
1233
167
		err := s.store.IncCounter(key)
1234
168
		c.Assert(err, IsNil)
1235
169
	}
1236
170
1237
171
	server, _ := s.prepareServer(c)
1238
172
1239
173
	expected := map[string]string{
1240
174
		"a:b": "2",
1241
175
		"a:*": "3",
1242
176
		"a":   "1",
1243
177
	}
1244
178
1245
179
	for counter, n := range expected {
1246
180
		req, err := http.NewRequest("GET", "/stats/counter/"+counter, nil)
1247
181
		c.Assert(err, IsNil)
1248
182
		rec := httptest.NewRecorder()
1249
183
		server.ServeHTTP(rec, req)
1250
184
1251
185
		data, err := ioutil.ReadAll(rec.Body)
1252
186
		c.Assert(string(data), Equals, n)
1253
187
1254
188
		c.Assert(rec.Header().Get("Content-Type"), Equals, "text/plain")
1255
189
		c.Assert(rec.Header().Get("Content-Length"), Equals, strconv.Itoa(len(n)))
1256
190
	}
1257
191
}
1258
192
1259
193
func (s *StoreSuite) TestBlitzKey(c *C) {
1260
194
	server, _ := s.prepareServer(c)
1261
195
1262
196
	// This is just a validation key to allow blitz.io to run
1263
197
	// performance tests against the site.
1264
198
	req, err := http.NewRequest("GET", "/mu-35700a31-6bf320ca-a800b670-05f845ee", nil)
1265
199
	c.Assert(err, IsNil)
1266
200
	rec := httptest.NewRecorder()
1267
201
	server.ServeHTTP(rec, req)
1268
202
1269
203
	data, err := ioutil.ReadAll(rec.Body)
1270
204
	c.Assert(string(data), Equals, "42")
1271
205
1272
206
	c.Assert(rec.Header().Get("Connection"), Equals, "close")
1273
207
	c.Assert(rec.Header().Get("Content-Type"), Equals, "text/plain")
1274
208
	c.Assert(rec.Header().Get("Content-Length"), Equals, "2")
1275
209
}
1276
210
0
1277
=== removed file 'store/store.go'
1278
--- store/store.go	2012-10-11 17:40:17 +0000
1279
+++ store/store.go	1970-01-01 00:00:00 +0000
1280
@@ -1,774 +0,0 @@
1281
1
// The store package is capable of storing and updating charms in a MongoDB
1282
2
// database, as well as maintaining further information about them such as
1283
3
// the VCS revision the charm was loaded from and the URLs for the charms.
1284
4
package store
1285
5
1286
6
import (
1287
7
	"crypto/sha256"
1288
8
	"encoding/hex"
1289
9
	"errors"
1290
10
	"fmt"
1291
11
	"hash"
1292
12
	"io"
1293
13
	"labix.org/v2/mgo"
1294
14
	"labix.org/v2/mgo/bson"
1295
15
	"launchpad.net/juju-core/charm"
1296
16
	"launchpad.net/juju-core/log"
1297
17
	"sort"
1298
18
	"strconv"
1299
19
	"sync"
1300
20
	"time"
1301
21
)
1302
22
1303
23
// The following MongoDB collections are currently used:
1304
24
//
1305
25
//     juju.events        - Log of events relating to the lifecycle of charms
1306
26
//     juju.charms        - Information about the stored charms
1307
27
//     juju.charmfs.*     - GridFS with the charm files
1308
28
//     juju.locks         - Has unique keys with url of updating charms
1309
29
//     juju.stat.counters - Counters for statistics
1310
30
//     juju.stat.tokens   - Tokens used in statistics counter keys
1311
31
1312
32
var (
1313
33
	ErrUpdateConflict  = errors.New("charm update in progress")
1314
34
	ErrRedundantUpdate = errors.New("charm is up-to-date")
1315
35
	ErrNotFound        = errors.New("entry not found")
1316
36
)
1317
37
1318
38
const (
1319
39
	UpdateTimeout = 600e9
1320
40
)
1321
41
1322
42
// Store holds a connection to a charm store.
1323
43
type Store struct {
1324
44
	session *storeSession
1325
45
1326
46
	// Cache for statistics key words (two generations).
1327
47
	cacheMu       sync.RWMutex
1328
48
	statsTokenNew map[string]int
1329
49
	statsTokenOld map[string]int
1330
50
}
1331
51
1332
52
// Open creates a new session with the store. It connects to the MongoDB
1333
53
// server at the given address (as expected by the Mongo function in the
1334
54
// labix.org/v2/mgo package).
1335
55
func Open(mongoAddr string) (store *Store, err error) {
1336
56
	log.Printf("store: Store opened. Connecting to: %s", mongoAddr)
1337
57
	store = &Store{}
1338
58
	session, err := mgo.Dial(mongoAddr)
1339
59
	if err != nil {
1340
60
		log.Printf("store: Error connecting to MongoDB: %v", err)
1341
61
		return nil, err
1342
62
	}
1343
63
1344
64
	store = &Store{session: &storeSession{session}}
1345
65
1346
66
	// Ignore error. It'll always fail after created.
1347
67
	// TODO Check the error once mgo hands it to us.
1348
68
	_ = store.session.DB("juju").Run(bson.D{{"create", "stat.counters"}, {"autoIndexId", false}}, nil)
1349
69
1350
70
	if err := store.ensureIndexes(); err != nil {
1351
71
		session.Close()
1352
72
		return nil, err
1353
73
	}
1354
74
1355
75
	// Put the used socket back in the pool.
1356
76
	session.Refresh()
1357
77
	return store, nil
1358
78
}
1359
79
1360
80
func (s *Store) ensureIndexes() error {
1361
81
	session := s.session
1362
82
	indexes := []struct {
1363
83
		c *mgo.Collection
1364
84
		i mgo.Index
1365
85
	}{{
1366
86
		session.StatCounters(),
1367
87
		mgo.Index{Key: []string{"k", "t"}, Unique: true},
1368
88
	}, {
1369
89
		session.StatTokens(),
1370
90
		mgo.Index{Key: []string{"t"}, Unique: true},
1371
91
	}, {
1372
92
		session.Charms(),
1373
93
		mgo.Index{Key: []string{"urls", "revision"}, Unique: true},
1374
94
	}, {
1375
95
		session.Events(),
1376
96
		mgo.Index{Key: []string{"urls", "digest"}},
1377
97
	}}
1378
98
	for _, idx := range indexes {
1379
99
		err := idx.c.EnsureIndex(idx.i)
1380
100
		if err != nil {
1381
101
			log.Printf("store: Error ensuring stat.counters index: %v", err)
1382
102
			return err
1383
103
		}
1384
104
	}
1385
105
	return nil
1386
106
}
1387
107
1388
108
// Close terminates the connection with the store.
1389
109
func (s *Store) Close() {
1390
110
	s.session.Close()
1391
111
}
1392
112
1393
113
// statsKey returns the compound statistics identifier that represents key.
1394
114
// If write is true, the identifier will be created if necessary.
1395
115
// Identifiers have a form similar to "ab:c:def:", where each section is a
1396
116
// base-32 number that represents the respective word in key. This form
1397
117
// allows efficiently indexing and searching for prefixes, while detaching
1398
118
// the key content and size from the actual words used in key.
1399
119
func (s *Store) statsKey(session *storeSession, key []string, write bool) (string, error) {
1400
120
	if len(key) == 0 {
1401
121
		return "", fmt.Errorf("store: empty statistics key")
1402
122
	}
1403
123
	tokens := session.StatTokens()
1404
124
	skey := make([]byte, 0, len(key)*4)
1405
125
	// Retry limit is mainly to prevent infinite recursion in edge cases,
1406
126
	// such as if the database is ever run in read-only mode.
1407
127
	// The logic below should deteministically stop in normal scenarios.
1408
128
	var err error
1409
129
	for i, retry := 0, 30; i < len(key) && retry > 0; retry-- {
1410
130
		err = nil
1411
131
		id, found := s.statsTokenId(key[i])
1412
132
		if !found {
1413
133
			var t struct {
1414
134
				Id    int    "_id"
1415
135
				Token string "t"
1416
136
			}
1417
137
			err = tokens.Find(bson.D{{"t", key[i]}}).One(&t)
1418
138
			if err == mgo.ErrNotFound {
1419
139
				if !write {
1420
140
					return "", ErrNotFound
1421
141
				}
1422
142
				t.Id, err = tokens.Count()
1423
143
				if err != nil {
1424
144
					continue
1425
145
				}
1426
146
				t.Id++
1427
147
				t.Token = key[i]
1428
148
				err = tokens.Insert(&t)
1429
149
			}
1430
150
			if err != nil {
1431
151
				continue
1432
152
			}
1433
153
			s.cacheStatsTokenId(t.Token, t.Id)
1434
154
			id = t.Id
1435
155
		}
1436
156
		skey = strconv.AppendInt(skey, int64(id), 32)
1437
157
		skey = append(skey, ':')
1438
158
		i++
1439
159
	}
1440
160
	if err != nil {
1441
161
		return "", err
1442
162
	}
1443
163
	return string(skey), nil
1444
164
}
1445
165
1446
166
const statsTokenCacheSize = 512
1447
167
1448
168
// cacheStatsTokenId adds the id for token into the cache.
1449
169
// The cache has two generations so that the least frequently used
1450
170
// tokens are evicted regularly.
1451
171
func (s *Store) cacheStatsTokenId(token string, id int) {
1452
172
	s.cacheMu.Lock()
1453
173
	defer s.cacheMu.Unlock()
1454
174
	// Can't possibly be >, but reviews want it for defensiveness.
1455
175
	if len(s.statsTokenNew) >= statsTokenCacheSize {
1456
176
		s.statsTokenOld = s.statsTokenNew
1457
177
		s.statsTokenNew = nil
1458
178
	}
1459
179
	if s.statsTokenNew == nil {
1460
180
		s.statsTokenNew = make(map[string]int, statsTokenCacheSize)
1461
181
	}
1462
182
	s.statsTokenNew[token] = id
1463
183
}
1464
184
1465
185
// statsTokenId returns the id for token from the cache, if found.
1466
186
func (s *Store) statsTokenId(token string) (id int, found bool) {
1467
187
	s.cacheMu.RLock()
1468
188
	id, found = s.statsTokenNew[token]
1469
189
	if found {
1470
190
		s.cacheMu.RUnlock()
1471
191
		return
1472
192
	}
1473
193
	id, found = s.statsTokenOld[token]
1474
194
	s.cacheMu.RUnlock()
1475
195
	if found {
1476
196
		s.cacheStatsTokenId(token, id)
1477
197
	}
1478
198
	return
1479
199
}
1480
200
1481
201
var counterEpoch = time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
1482
202
1483
203
// IncCounter increases by one the counter associated with the composed key.
1484
204
func (s *Store) IncCounter(key []string) error {
1485
205
	session := s.session.Copy()
1486
206
	defer session.Close()
1487
207
1488
208
	skey, err := s.statsKey(session, key, true)
1489
209
	if err != nil {
1490
210
		return err
1491
211
	}
1492
212
1493
213
	t := time.Now().UTC()
1494
214
	// Round to the start of the minute so we get one document per minute at most.
1495
215
	t = t.Add(-time.Duration(t.Second()) * time.Second)
1496
216
	counters := session.StatCounters()
1497
217
	_, err = counters.Upsert(bson.D{{"k", skey}, {"t", int32(t.Unix() - counterEpoch)}}, bson.D{{"$inc", bson.D{{"c", 1}}}})
1498
218
	return err
1499
219
}
1500
220
1501
221
// SumCounter returns the sum of all the counters that exactly match key,
1502
222
// or that are prefixed by it if prefix is true.
1503
223
func (s *Store) SumCounter(key []string, prefix bool) (count int64, err error) {
1504
224
	session := s.session.Copy()
1505
225
	defer session.Close()
1506
226
1507
227
	skey, err := s.statsKey(session, key, false)
1508
228
	if err == ErrNotFound {
1509
229
		return 0, nil
1510
230
	}
1511
231
	if err != nil {
1512
232
		return 0, err
1513
233
	}
1514
234
1515
235
	var regex string
1516
236
	if prefix {
1517
237
		regex = "^" + skey
1518
238
	} else {
1519
239
		regex = "^" + skey + "$"
1520
240
	}
1521
241
1522
242
	job := mgo.MapReduce{
1523
243
		Map:    "function() { emit('count', this.c); }",
1524
244
		Reduce: "function(key, values) { return Array.sum(values); }",
1525
245
	}
1526
246
	var result []struct{ Value int64 }
1527
247
	counters := session.StatCounters()
1528
248
	_, err = counters.Find(bson.D{{"k", bson.D{{"$regex", regex}}}}).MapReduce(&job, &result)
1529
249
	if len(result) > 0 {
1530
250
		return result[0].Value, err
1531
251
	}
1532
252
	return 0, err
1533
253
}
1534
254
1535
255
// A CharmPublisher is responsible for importing a charm dir onto the store.
1536
256
type CharmPublisher struct {
1537
257
	revision int
1538
258
	w        *charmWriter
1539
259
}
1540
260
1541
261
// Revision returns the revision that will be assigned to the published charm.
1542
262
func (p *CharmPublisher) Revision() int {
1543
263
	return p.revision
1544
264
}
1545
265
1546
266
// CharmDir matches the part of the interface of *charm.Dir that is necessary
1547
267
// to publish a charm. Using this interface rather than *charm.Dir directly
1548
268
// makes testing some aspects of the store possible.
1549
269
type CharmDir interface {
1550
270
	Meta() *charm.Meta
1551
271
	Config() *charm.Config
1552
272
	SetRevision(revision int)
1553
273
	BundleTo(w io.Writer) error
1554
274
}
1555
275
1556
276
// Statically ensure that *charm.Dir is indeed a CharmDir.
1557
277
var _ CharmDir = (*charm.Dir)(nil)
1558
278
1559
279
// Publish bundles charm and writes it to the store. The written charm
1560
280
// bundle will have its revision set to the result of Revision.
1561
281
// Publish must be called only once for a CharmPublisher.
1562
282
func (p *CharmPublisher) Publish(charm CharmDir) error {
1563
283
	w := p.w
1564
284
	if w == nil {
1565
285
		panic("CharmPublisher already published a charm")
1566
286
	}
1567
287
	p.w = nil
1568
288
	w.charm = charm
1569
289
	// TODO: Refactor to BundleTo(w, revision)
1570
290
	charm.SetRevision(p.revision)
1571
291
	err := charm.BundleTo(w)
1572
292
	if err == nil {
1573
293
		err = w.finish()
1574
294
	} else {
1575
295
		w.abort()
1576
296
	}
1577
297
	return err
1578
298
}
1579
299
1580
300
// CharmPublisher returns a new CharmPublisher for importing a charm that
1581
301
// will be made available in the store at all of the provided URLs.
1582
302
// The digest parameter must contain the unique identifier that
1583
303
// represents the charm data being imported (e.g. the VCS revision sha1).
1584
304
// ErrRedundantUpdate is returned if all of the provided urls are
1585
305
// already associated to that digest.
1586
306
func (s *Store) CharmPublisher(urls []*charm.URL, digest string) (p *CharmPublisher, err error) {
1587
307
	log.Printf("store: Trying to add charms %v with key %q...", urls, digest)
1588
308
	if err = mustLackRevision("CharmPublisher", urls...); err != nil {
1589
309
		return
1590
310
	}
1591
311
	session := s.session.Copy()
1592
312
	defer session.Close()
1593
313
1594
314
	maxRev := -1
1595
315
	newKey := false
1596
316
	charms := session.Charms()
1597
317
	doc := charmDoc{}
1598
318
	for i := range urls {
1599
319
		urlStr := urls[i].String()
1600
320
		err = charms.Find(bson.D{{"urls", urlStr}}).Sort("-revision").One(&doc)
1601
321
		if err == mgo.ErrNotFound {
1602
322
			log.Printf("store: Charm %s not yet in the store.", urls[i])
1603
323
			newKey = true
1604
324
			continue
1605
325
		}
1606
326
		if doc.Digest != digest {
1607
327
			log.Printf("store: Charm %s is out of date with revision key %q.", urlStr, digest)
1608
328
			newKey = true
1609
329
		}
1610
330
		if err != nil {
1611
331
			log.Printf("store: Unknown error looking for charm %s: %s", urlStr, err)
1612
332
			return
1613
333
		}
1614
334
		if doc.Revision > maxRev {
1615
335
			maxRev = doc.Revision
1616
336
		}
1617
337
	}
1618
338
	if !newKey {
1619
339
		log.Printf("store: All charms have revision key %q. Nothing to update.", digest)
1620
340
		err = ErrRedundantUpdate
1621
341
		return
1622
342
	}
1623
343
	revision := maxRev + 1
1624
344
	log.Printf("store: Preparing writer to add charms with revision %d.", revision)
1625
345
	w := &charmWriter{
1626
346
		store:    s,
1627
347
		urls:     urls,
1628
348
		revision: revision,
1629
349
		digest:   digest,
1630
350
	}
1631
351
	return &CharmPublisher{revision, w}, nil
1632
352
}
1633
353
1634
354
// charmWriter is an io.Writer that writes charm bundles to the charms GridFS.
1635
355
type charmWriter struct {
1636
356
	store    *Store
1637
357
	session  *storeSession
1638
358
	file     *mgo.GridFile
1639
359
	sha256   hash.Hash
1640
360
	charm    CharmDir
1641
361
	urls     []*charm.URL
1642
362
	revision int
1643
363
	digest   string
1644
364
}
1645
365
1646
366
// Write creates an entry in the charms GridFS when first called,
1647
367
// and streams all written data into it.
1648
368
func (w *charmWriter) Write(data []byte) (n int, err error) {
1649
369
	if w.file == nil {
1650
370
		w.session = w.store.session.Copy()
1651
371
		w.file, err = w.session.CharmFS().Create("")
1652
372
		if err != nil {
1653
373
			log.Printf("store: Failed to create GridFS file: %v", err)
1654
374
			return 0, err
1655
375
		}
1656
376
		w.sha256 = sha256.New()
1657
377
		log.Printf("store: Creating GridFS file with id %q...", w.file.Id().(bson.ObjectId).Hex())
1658
378
	}
1659
379
	_, err = w.sha256.Write(data)
1660
380
	if err != nil {
1661
381
		panic("hash.Hash should never error")
1662
382
	}
1663
383
	return w.file.Write(data)
1664
384
}
1665
385
1666
386
// abort cancels the charm writing.
1667
387
func (w *charmWriter) abort() {
1668
388
	if w.file != nil {
1669
389
		// Ignore error. Already aborting due to a preceding bad situation
1670
390
		// elsewhere. This error is not important right now.
1671
391
		_ = w.file.Close()
1672
392
		w.session.Close()
1673
393
	}
1674
394
}
1675
395
1676
396
// finish completes the charm writing process and inserts the final metadata.
1677
397
// After it completes the charm will be available for consumption.
1678
398
func (w *charmWriter) finish() error {
1679
399
	if w.file == nil {
1680
400
		return nil
1681
401
	}
1682
402
	defer w.session.Close()
1683
403
	id := w.file.Id()
1684
404
	size := w.file.Size()
1685
405
	err := w.file.Close()
1686
406
	if err != nil {
1687
407
		log.Printf("store: Failed to close GridFS file: %v", err)
1688
408
		return err
1689
409
	}
1690
410
	charms := w.session.Charms()
1691
411
	sha256 := hex.EncodeToString(w.sha256.Sum(nil))
1692
412
	charm := charmDoc{
1693
413
		w.urls,
1694
414
		w.revision,
1695
415
		w.digest,
1696
416
		sha256,
1697
417
		size,
1698
418
		id.(bson.ObjectId),
1699
419
		w.charm.Meta(),
1700
420
		w.charm.Config(),
1701
421
	}
1702
422
	if err = charms.Insert(&charm); err != nil {
1703
423
		err = maybeConflict(err)
1704
424
		log.Printf("store: Failed to insert new revision of charm %v: %v", w.urls, err)
1705
425
		return err
1706
426
	}
1707
427
	return nil
1708
428
}
1709
429
1710
430
type CharmInfo struct {
1711
431
	revision int
1712
432
	digest   string
1713
433
	sha256   string
1714
434
	size     int64
1715
435
	fileId   bson.ObjectId
1716
436
	meta     *charm.Meta
1717
437
	config   *charm.Config
1718
438
}
1719
439
1720
440
// Statically ensure CharmInfo is a charm.Charm.
1721
441
var _ charm.Charm = (*CharmInfo)(nil)
1722
442
1723
443
// Revision returns the store charm's revision.
1724
444
func (ci *CharmInfo) Revision() int {
1725
445
	return ci.revision
1726
446
}
1727
447
1728
448
// BundleSha256 returns the sha256 checksum for the stored charm bundle.
1729
449
func (ci *CharmInfo) BundleSha256() string {
1730
450
	return ci.sha256
1731
451
}
1732
452
1733
453
// BundleSize returns the size for the stored charm bundle.
1734
454
func (ci *CharmInfo) BundleSize() int64 {
1735
455
	return ci.size
1736
456
}
1737
457
1738
458
// Digest returns the unique identifier that represents the charm
1739
459
// data imported. This is typically set to the VCS revision digest.
1740
460
func (ci *CharmInfo) Digest() string {
1741
461
	return ci.digest
1742
462
}
1743
463
1744
464
// Meta returns the charm.Meta details for the stored charm.
1745
465
func (ci *CharmInfo) Meta() *charm.Meta {
1746
466
	return ci.meta
1747
467
}
1748
468
1749
469
// Config returns the charm.Config details for the stored charm.
1750
470
func (ci *CharmInfo) Config() *charm.Config {
1751
471
	return ci.config
1752
472
}
1753
473
1754
474
// CharmInfo retrieves the CharmInfo value for the charm at url.
1755
475
func (s *Store) CharmInfo(url *charm.URL) (info *CharmInfo, err error) {
1756
476
	session := s.session.Copy()
1757
477
	defer session.Close()
1758
478
1759
479
	log.Debugf("store: Retrieving charm info for %s", url)
1760
480
	rev := url.Revision
1761
481
	url = url.WithRevision(-1)
1762
482
1763
483
	charms := session.Charms()
1764
484
	var cdoc charmDoc
1765
485
	var qdoc interface{}
1766
486
	if rev == -1 {
1767
487
		qdoc = bson.D{{"urls", url}}
1768
488
	} else {
1769
489
		qdoc = bson.D{{"urls", url}, {"revision", rev}}
1770
490
	}
1771
491
	err = charms.Find(qdoc).Sort("-revision").One(&cdoc)
1772
492
	if err != nil {
1773
493
		log.Printf("store: Failed to find charm %s: %v", url, err)
1774
494
		return nil, ErrNotFound
1775
495
	}
1776
496
	info = &CharmInfo{
1777
497
		cdoc.Revision,
1778
498
		cdoc.Digest,
1779
499
		cdoc.Sha256,
1780
500
		cdoc.Size,
1781
501
		cdoc.FileId,
1782
502
		cdoc.Meta,
1783
503
		cdoc.Config,
1784
504
	}
1785
505
	return info, nil
1786
506
}
1787
507
1788
508
// OpenCharm opens for reading via rc the charm currently available at url.
1789
509
// rc must be closed after dealing with it or resources will leak.
1790
510
func (s *Store) OpenCharm(url *charm.URL) (info *CharmInfo, rc io.ReadCloser, err error) {
1791
511
	log.Debugf("store: Opening charm %s", url)
1792
512
	info, err = s.CharmInfo(url)
1793
513
	if err != nil {
1794
514
		return nil, nil, err
1795
515
	}
1796
516
	session := s.session.Copy()
1797
517
	file, err := session.CharmFS().OpenId(info.fileId)
1798
518
	if err != nil {
1799
519
		log.Printf("store: Failed to open GridFS file for charm %s: %v", url, err)
1800
520
		session.Close()
1801
521
		return nil, nil, err
1802
522
	}
1803
523
	rc = &reader{session, file}
1804
524
	return
1805
525
}
1806
526
1807
527
type reader struct {
1808
528
	session *storeSession
1809
529
	file    *mgo.GridFile
1810
530
}
1811
531
1812
532
// Read consumes data from the opened charm.
1813
533
func (r *reader) Read(buf []byte) (n int, err error) {
1814
534
	return r.file.Read(buf)
1815
535
}
1816
536
1817
537
// Close closes the opened charm and frees associated resources.
1818
538
func (r *reader) Close() error {
1819
539
	err := r.file.Close()
1820
540
	r.session.Close()
1821
541
	return err
1822
542
}
1823
543
1824
544
// charmDoc represents the document stored in MongoDB for a charm.
1825
545
type charmDoc struct {
1826
546
	URLs     []*charm.URL
1827
547
	Revision int
1828
548
	Digest   string
1829
549
	Sha256   string
1830
550
	Size     int64
1831
551
	FileId   bson.ObjectId
1832
552
	Meta     *charm.Meta
1833
553
	Config   *charm.Config
1834
554
}
1835
555
1836
556
// LockUpdates acquires a server-side lock for updating a single charm
1837
557
// that is supposed to be made available in all of the provided urls.
1838
558
// If the lock can't be acquired in any of the urls, an error will be
1839
559
// immediately returned.
1840
560
// In the usual case, any locking done is undone when an error happens,
1841
561
// or when l.Unlock is called. If something else goes wrong, the locks
1842
562
// will also expire after the period defined in UpdateTimeout.
1843
563
func (s *Store) LockUpdates(urls []*charm.URL) (l *UpdateLock, err error) {
1844
564
	session := s.session.Copy()
1845
565
	keys := make([]string, len(urls))
1846
566
	for i := range urls {
1847
567
		keys[i] = urls[i].String()
1848
568
	}
1849
569
	sort.Strings(keys)
1850
570
	l = &UpdateLock{keys, session.Locks(), bson.Now()}
1851
571
	if err = l.tryLock(); err != nil {
1852
572
		session.Close()
1853
573
		return nil, err
1854
574
	}
1855
575
	return l, nil
1856
576
}
1857
577
1858
578
// UpdateLock represents an acquired update lock over a set of charm URLs.
1859
579
type UpdateLock struct {
1860
580
	keys  []string
1861
581
	locks *mgo.Collection
1862
582
	time  time.Time
1863
583
}
1864
584
1865
585
// Unlock removes the previously acquired server-side lock that prevents
1866
586
// other processes from attempting to update a set of charm URLs.
1867
587
func (l *UpdateLock) Unlock() {
1868
588
	log.Debugf("store: Unlocking charms for future updates: %v", l.keys)
1869
589
	defer l.locks.Database.Session.Close()
1870
590
	for i := len(l.keys) - 1; i >= 0; i-- {
1871
591
		// Using time below ensures only the proper lock is removed.
1872
592
		// Can't do much about errors here. Locks will expire anyway.
1873
593
		l.locks.Remove(bson.D{{"_id", l.keys[i]}, {"time", l.time}})
1874
594
	}
1875
595
}
1876
596
1877
597
// tryLock tries locking l.keys, one at a time, and succeeds only if it
1878
598
// can lock all of them in order. The keys should be pre-sorted so that
1879
599
// two-way conflicts can't happen. If any of the keys fail to be locked,
1880
600
// and expiring the old lock doesn't work, tryLock undoes all previous
1881
601
// locks and aborts with an error.
1882
602
func (l *UpdateLock) tryLock() error {
1883
603
	for i, key := range l.keys {
1884
604
		log.Debugf("store: Trying to lock charm %s for updates...", key)
1885
605
		doc := bson.D{{"_id", key}, {"time", l.time}}
1886
606
		err := l.locks.Insert(doc)
1887
607
		if err == nil {
1888
608
			log.Debugf("store: Charm %s is now locked for updates.", key)
1889
609
			continue
1890
610
		}
1891
611
		if lerr, ok := err.(*mgo.LastError); ok && lerr.Code == 11000 {
1892
612
			log.Debugf("store: Charm %s is locked. Trying to expire lock.", key)
1893
613
			l.tryExpire(key)
1894
614
			err = l.locks.Insert(doc)
1895
615
			if err == nil {
1896
616
				log.Debugf("store: Charm %s is now locked for updates.", key)
1897
617
				continue
1898
618
			}
1899
619
		}
1900
620
		// Couldn't lock everyone. Undo previous locks.
1901
621
		for j := i - 1; j >= 0; j-- {
1902
622
			// Using time below should be unnecessary, but it's an extra check.
1903
623
			// Can't do anything about errors here. Lock will expire anyway.
1904
624
			l.locks.Remove(bson.D{{"_id", l.keys[j]}, {"time", l.time}})
1905
625
		}
1906
626
		err = maybeConflict(err)
1907
627
		log.Printf("store: Can't lock charms %v for updating: %v", l.keys, err)
1908
628
		return err
1909
629
	}
1910
630
	return nil
1911
631
}
1912
632
1913
633
// tryExpire attempts to remove outdated locks from the database.
1914
634
func (l *UpdateLock) tryExpire(key string) {
1915
635
	// Ignore errors. If nothing happens the key will continue locked.
1916
636
	l.locks.Remove(bson.D{{"_id", key}, {"time", bson.D{{"$lt", bson.Now().Add(-UpdateTimeout)}}}})
1917
637
}
1918
638
1919
639
// maybeConflict returns an ErrUpdateConflict if err is a mgo
1920
640
// insert conflict LastError, or err itself otherwise.
1921
641
func maybeConflict(err error) error {
1922
642
	if lerr, ok := err.(*mgo.LastError); ok && lerr.Code == 11000 {
1923
643
		return ErrUpdateConflict
1924
644
	}
1925
645
	return err
1926
646
}
1927
647
1928
648
// storeSession wraps a mgo.Session ands adds a few convenience methods.
1929
649
type storeSession struct {
1930
650
	*mgo.Session
1931
651
}
1932
652
1933
653
// Copy copies the storeSession and its underlying mgo session.
1934
654
func (s *storeSession) Copy() *storeSession {
1935
655
	return &storeSession{s.Session.Copy()}
1936
656
}
1937
657
1938
658
// Charms returns the mongo collection where charms are stored.
1939
659
func (s *storeSession) Charms() *mgo.Collection {
1940
660
	return s.DB("juju").C("charms")
1941
661
}
1942
662
1943
663
// Charms returns a mgo.GridFS to read and write charms.
1944
664
func (s *storeSession) CharmFS() *mgo.GridFS {
1945
665
	return s.DB("juju").GridFS("charmfs")
1946
666
}
1947
667
1948
668
// Events returns the mongo collection where charm events are stored.
1949
669
func (s *storeSession) Events() *mgo.Collection {
1950
670
	return s.DB("juju").C("events")
1951
671
}
1952
672
1953
673
// Locks returns the mongo collection where charm locks are stored.
1954
674
func (s *storeSession) Locks() *mgo.Collection {
1955
675
	return s.DB("juju").C("locks")
1956
676
}
1957
677
1958
678
// StatTokens returns the mongo collection for storing key tokens
1959
679
// for statistics collection.
1960
680
func (s *storeSession) StatTokens() *mgo.Collection {
1961
681
	return s.DB("juju").C("stat.tokens")
1962
682
}
1963
683
1964
684
// StatCounters returns the mongo collection for counter values.
1965
685
func (s *storeSession) StatCounters() *mgo.Collection {
1966
686
	return s.DB("juju").C("stat.counters")
1967
687
}
1968
688
1969
689
type CharmEventKind int
1970
690
1971
691
const (
1972
692
	EventPublished CharmEventKind = iota + 1
1973
693
	EventPublishError
1974
694
1975
695
	EventKindCount
1976
696
)
1977
697
1978
698
func (k CharmEventKind) String() string {
1979
699
	switch k {
1980
700
	case EventPublished:
1981
701
		return "published"
1982
702
	case EventPublishError:
1983
703
		return "publish-error"
1984
704
	}
1985
705
	panic(fmt.Errorf("unknown charm event kind %d", k))
1986
706
}
1987
707
1988
708
// CharmEvent is a record for an event relating to one or more charm URLs.
1989
709
type CharmEvent struct {
1990
710
	Kind     CharmEventKind
1991
711
	Digest   string
1992
712
	Revision int
1993
713
	URLs     []*charm.URL
1994
714
	Errors   []string `bson:",omitempty"`
1995
715
	Warnings []string `bson:",omitempty"`
1996
716
	Time     time.Time
1997
717
}
1998
718
1999
719
// LogCharmEvent records an event related to one or more charm URLs.
2000
720
func (s *Store) LogCharmEvent(event *CharmEvent) (err error) {
2001
721
	log.Printf("store: Adding charm event for %v with key %q: %s", event.URLs, event.Digest, event.Kind)
2002
722
	if err = mustLackRevision("LogCharmEvent", event.URLs...); err != nil {
2003
723
		return
2004
724
	}
2005
725
	session := s.session.Copy()
2006
726
	defer session.Close()
2007
727
	if event.Kind == 0 || event.Digest == "" || len(event.URLs) == 0 {
2008
728
		return fmt.Errorf("LogCharmEvent: need valid Kind, Digest and URLs")
2009
729
	}
2010
730
	if event.Time.IsZero() {
2011
731
		event.Time = time.Now()
2012
732
	}
2013
733
	events := session.Events()
2014
734
	return events.Insert(event)
2015
735
}
2016
736
2017
737
// CharmEvent returns the most recent event associated with url
2018
738
// and digest.  If the specified event isn't found the error
2019
739
// ErrUnknownChange will be returned.
2020
740
func (s *Store) CharmEvent(url *charm.URL, digest string) (*CharmEvent, error) {
2021
741
	// TODO: It'd actually make sense to find the charm event after the
2022
742
	// revision id, but since we don't care about that now, just make sure
2023
743
	// we don't write bad code.
2024
744
	if err := mustLackRevision("CharmEvent", url); err != nil {
2025
745
		return nil, err
2026
746
	}
2027
747
	session := s.session.Copy()
2028
748
	defer session.Close()
2029
749
2030
750
	events := session.Events()
2031
751
	event := &CharmEvent{Digest: digest}
2032
752
	query := events.Find(bson.D{{"urls", url}, {"digest", digest}})
2033
753
	query.Sort("-time")
2034
754
	err := query.One(&event)
2035
755
	if err == mgo.ErrNotFound {
2036
756
		return nil, ErrNotFound
2037
757
	}
2038
758
	if err != nil {
2039
759
		return nil, err
2040
760
	}
2041
761
	return event, nil
2042
762
}
2043
763
2044
764
// mustLackRevision returns an error if any of the urls has a revision.
2045
765
func mustLackRevision(context string, urls ...*charm.URL) error {
2046
766
	for _, url := range urls {
2047
767
		if url.Revision != -1 {
2048
768
			err := fmt.Errorf("%s: got charm URL with revision: %s", context, url)
2049
769
			log.Printf("store: %v", err)
2050
770
			return err
2051
771
		}
2052
772
	}
2053
773
	return nil
2054
774
}
2055
775
0
2056
=== removed file 'store/store_test.go'
2057
--- store/store_test.go	2012-11-20 07:18:32 +0000
2058
+++ store/store_test.go	1970-01-01 00:00:00 +0000
2059
@@ -1,608 +0,0 @@
2060
1
package store_test
2061
2
2062
3
import (
2063
4
	"fmt"
2064
5
	"io"
2065
6
	"io/ioutil"
2066
7
	"labix.org/v2/mgo/bson"
2067
8
	. "launchpad.net/gocheck"
2068
9
	"launchpad.net/juju-core/charm"
2069
10
	"launchpad.net/juju-core/log"
2070
11
	"launchpad.net/juju-core/store"
2071
12
	"launchpad.net/juju-core/testing"
2072
13
	"strconv"
2073
14
	"sync"
2074
15
	stdtesting "testing"
2075
16
	"time"
2076
17
)
2077
18
2078
19
func Test(t *stdtesting.T) {
2079
20
	TestingT(t)
2080
21
}
2081
22
2082
23
var _ = Suite(&StoreSuite{})
2083
24
var _ = Suite(&TrivialSuite{})
2084
25
2085
26
type StoreSuite struct {
2086
27
	MgoSuite
2087
28
	testing.HTTPSuite
2088
29
	store *store.Store
2089
30
}
2090
31
2091
32
type TrivialSuite struct{}
2092
33
2093
34
func (s *StoreSuite) SetUpSuite(c *C) {
2094
35
	s.MgoSuite.SetUpSuite(c)
2095
36
	s.HTTPSuite.SetUpSuite(c)
2096
37
}
2097
38
2098
39
func (s *StoreSuite) TearDownSuite(c *C) {
2099
40
	s.HTTPSuite.TearDownSuite(c)
2100
41
	s.MgoSuite.TearDownSuite(c)
2101
42
}
2102
43
2103
44
func (s *StoreSuite) SetUpTest(c *C) {
2104
45
	s.MgoSuite.SetUpTest(c)
2105
46
	var err error
2106
47
	s.store, err = store.Open(s.Addr)
2107
48
	c.Assert(err, IsNil)
2108
49
	log.Target = c
2109
50
	log.Debug = true
2110
51
}
2111
52
2112
53
func (s *StoreSuite) TearDownTest(c *C) {
2113
54
	s.HTTPSuite.TearDownTest(c)
2114
55
	if s.store != nil {
2115
56
		s.store.Close()
2116
57
	}
2117
58
	s.MgoSuite.TearDownTest(c)
2118
59
}
2119
60
2120
61
// FakeCharmDir is a charm that implements the interface that the
2121
62
// store publisher cares about.
2122
63
type FakeCharmDir struct {
2123
64
	revision interface{} // so we can tell if it's not set.
2124
65
	error    string
2125
66
}
2126
67
2127
68
func (d *FakeCharmDir) Meta() *charm.Meta {
2128
69
	return &charm.Meta{
2129
70
		Name:        "fakecharm",
2130
71
		Summary:     "Fake charm for testing purposes.",
2131
72
		Description: "This is a fake charm for testing purposes.\n",
2132
73
		Provides:    make(map[string]charm.Relation),
2133
74
		Requires:    make(map[string]charm.Relation),
2134
75
		Peers:       make(map[string]charm.Relation),
2135
76
	}
2136
77
}
2137
78
2138
79
func (d *FakeCharmDir) Config() *charm.Config {
2139
80
	return &charm.Config{make(map[string]charm.Option)}
2140
81
}
2141
82
2142
83
func (d *FakeCharmDir) SetRevision(revision int) {
2143
84
	d.revision = revision
2144
85
}
2145
86
2146
87
func (d *FakeCharmDir) BundleTo(w io.Writer) error {
2147
88
	if d.error == "beforeWrite" {
2148
89
		return fmt.Errorf(d.error)
2149
90
	}
2150
91
	_, err := w.Write([]byte(fmt.Sprintf("charm-revision-%v", d.revision)))
2151
92
	if d.error == "afterWrite" {
2152
93
		return fmt.Errorf(d.error)
2153
94
	}
2154
95
	return err
2155
96
}
2156
97
2157
98
func (s *StoreSuite) TestCharmPublisherWithRevisionedURL(c *C) {
2158
99
	urls := []*charm.URL{charm.MustParseURL("cs:oneiric/wordpress-0")}
2159
100
	pub, err := s.store.CharmPublisher(urls, "some-digest")
2160
101
	c.Assert(err, ErrorMatches, "CharmPublisher: got charm URL with revision: cs:oneiric/wordpress-0")
2161
102
	c.Assert(pub, IsNil)
2162
103
}
2163
104
2164
105
func (s *StoreSuite) TestCharmPublisher(c *C) {
2165
106
	urlA := charm.MustParseURL("cs:oneiric/wordpress-a")
2166
107
	urlB := charm.MustParseURL("cs:oneiric/wordpress-b")
2167
108
	urls := []*charm.URL{urlA, urlB}
2168
109
2169
110
	pub, err := s.store.CharmPublisher(urls, "some-digest")
2170
111
	c.Assert(err, IsNil)
2171
112
	c.Assert(pub.Revision(), Equals, 0)
2172
113
2173
114
	err = pub.Publish(testing.Charms.ClonedDir(c.MkDir(), "series", "dummy"))
2174
115
	c.Assert(err, IsNil)
2175
116
2176
117
	for _, url := range urls {
2177
118
		info, rc, err := s.store.OpenCharm(url)
2178
119
		c.Assert(err, IsNil)
2179
120
		c.Assert(info.Revision(), Equals, 0)
2180
121
		c.Assert(info.Digest(), Equals, "some-digest")
2181
122
		data, err := ioutil.ReadAll(rc)
2182
123
		c.Check(err, IsNil)
2183
124
		err = rc.Close()
2184
125
		c.Assert(err, IsNil)
2185
126
		bundle, err := charm.ReadBundleBytes(data)
2186
127
		c.Assert(err, IsNil)
2187
128
2188
129
		// The same information must be available by reading the
2189
130
		// full charm data...
2190
131
		c.Assert(bundle.Meta().Name, Equals, "dummy")
2191
132
		c.Assert(bundle.Config().Options["title"].Default, Equals, "My Title")
2192
133
2193
134
		// ... and the queriable details.
2194
135
		c.Assert(info.Meta().Name, Equals, "dummy")
2195
136
		c.Assert(info.Config().Options["title"].Default, Equals, "My Title")
2196
137
2197
138
		info2, err := s.store.CharmInfo(url)
2198
139
		c.Assert(err, IsNil)
2199
140
		c.Assert(info2, DeepEquals, info)
2200
141
	}
2201
142
}
2202
143
2203
144
func (s *StoreSuite) TestCharmPublishError(c *C) {
2204
145
	url := charm.MustParseURL("cs:oneiric/wordpress")
2205
146
	urls := []*charm.URL{url}
2206
147
2207
148
	// Publish one successfully to bump the revision so we can
2208
149
	// make sure it is being correctly set below.
2209
150
	pub, err := s.store.CharmPublisher(urls, "one-digest")
2210
151
	c.Assert(err, IsNil)
2211
152
	c.Assert(pub.Revision(), Equals, 0)
2212
153
	err = pub.Publish(&FakeCharmDir{})
2213
154
	c.Assert(err, IsNil)
2214
155
2215
156
	pub, err = s.store.CharmPublisher(urls, "another-digest")
2216
157
	c.Assert(err, IsNil)
2217
158
	c.Assert(pub.Revision(), Equals, 1)
2218
159
	err = pub.Publish(&FakeCharmDir{error: "beforeWrite"})
2219
160
	c.Assert(err, ErrorMatches, "beforeWrite")
2220
161
2221
162
	pub, err = s.store.CharmPublisher(urls, "another-digest")
2222
163
	c.Assert(err, IsNil)
2223
164
	c.Assert(pub.Revision(), Equals, 1)
2224
165
	err = pub.Publish(&FakeCharmDir{error: "afterWrite"})
2225
166
	c.Assert(err, ErrorMatches, "afterWrite")
2226
167
2227
168
	// Still at the original charm revision that succeeded first.
2228
169
	info, err := s.store.CharmInfo(url)
2229
170
	c.Assert(err, IsNil)
2230
171
	c.Assert(info.Revision(), Equals, 0)
2231
172
	c.Assert(info.Digest(), Equals, "one-digest")
2232
173
}
2233
174
2234
175
func (s *StoreSuite) TestCharmInfoNotFound(c *C) {
2235
176
	info, err := s.store.CharmInfo(charm.MustParseURL("cs:oneiric/wordpress"))
2236
177
	c.Assert(err, Equals, store.ErrNotFound)
2237
178
	c.Assert(info, IsNil)
2238
179
}
2239
180
2240
181
func (s *StoreSuite) TestRevisioning(c *C) {
2241
182
	urlA := charm.MustParseURL("cs:oneiric/wordpress-a")
2242
183
	urlB := charm.MustParseURL("cs:oneiric/wordpress-b")
2243
184
	urls := []*charm.URL{urlA, urlB}
2244
185
2245
186
	tests := []struct {
2246
187
		urls []*charm.URL
2247
188
		data string
2248
189
	}{
2249
190
		{urls[0:], "charm-revision-0"},
2250
191
		{urls[1:], "charm-revision-1"},
2251
192
		{urls[0:], "charm-revision-2"},
2252
193
	}
2253
194
2254
195
	for i, t := range tests {
2255
196
		pub, err := s.store.CharmPublisher(t.urls, fmt.Sprintf("digest-%d", i))
2256
197
		c.Assert(err, IsNil)
2257
198
		c.Assert(pub.Revision(), Equals, i)
2258
199
2259
200
		err = pub.Publish(&FakeCharmDir{})
2260
201
		c.Assert(err, IsNil)
2261
202
	}
2262
203
2263
204
	for i, t := range tests {
2264
205
		for _, url := range t.urls {
2265
206
			url = url.WithRevision(i)
2266
207
			info, rc, err := s.store.OpenCharm(url)
2267
208
			c.Assert(err, IsNil)
2268
209
			data, err := ioutil.ReadAll(rc)
2269
210
			cerr := rc.Close()
2270
211
			c.Assert(info.Revision(), Equals, i)
2271
212
			c.Assert(url.Revision, Equals, i) // Untouched.
2272
213
			c.Assert(cerr, IsNil)
2273
214
			c.Assert(string(data), Equals, string(t.data))
2274
215
			c.Assert(err, IsNil)
2275
216
		}
2276
217
	}
2277
218
2278
219
	info, rc, err := s.store.OpenCharm(urlA.WithRevision(1))
2279
220
	c.Assert(err, Equals, store.ErrNotFound)
2280
221
	c.Assert(info, IsNil)
2281
222
	c.Assert(rc, IsNil)
2282
223
}
2283
224
2284
225
func (s *StoreSuite) TestLockUpdates(c *C) {
2285
226
	urlA := charm.MustParseURL("cs:oneiric/wordpress-a")
2286
227
	urlB := charm.MustParseURL("cs:oneiric/wordpress-b")
2287
228
	urls := []*charm.URL{urlA, urlB}
2288
229
2289
230
	// Lock update of just B to force a partial conflict.
2290
231
	lock1, err := s.store.LockUpdates(urls[1:])
2291
232
	c.Assert(err, IsNil)
2292
233
2293
234
	// Partially conflicts with locked update above.
2294
235
	lock2, err := s.store.LockUpdates(urls)
2295
236
	c.Check(err, Equals, store.ErrUpdateConflict)
2296
237
	c.Check(lock2, IsNil)
2297
238
2298
239
	lock1.Unlock()
2299
240
2300
241
	// Trying again should work since lock1 was released.
2301
242
	lock3, err := s.store.LockUpdates(urls)
2302
243
	c.Assert(err, IsNil)
2303
244
	lock3.Unlock()
2304
245
}
2305
246
2306
247
func (s *StoreSuite) TestLockUpdatesExpires(c *C) {
2307
248
	urlA := charm.MustParseURL("cs:oneiric/wordpress-a")
2308
249
	urlB := charm.MustParseURL("cs:oneiric/wordpress-b")
2309
250
	urls := []*charm.URL{urlA, urlB}
2310
251
2311
252
	// Initiate an update of B only to force a partial conflict.
2312
253
	lock1, err := s.store.LockUpdates(urls[1:])
2313
254
	c.Assert(err, IsNil)
2314
255
2315
256
	// Hack time to force an expiration.
2316
257
	locks := s.Session.DB("juju").C("locks")
2317
258
	selector := bson.M{"_id": urlB.String()}
2318
259
	update := bson.M{"time": bson.Now().Add(-store.UpdateTimeout - 10e9)}
2319
260
	err = locks.Update(selector, update)
2320
261
	c.Check(err, IsNil)
2321
262
2322
263
	// Works due to expiration of previous lock.
2323
264
	lock2, err := s.store.LockUpdates(urls)
2324
265
	c.Assert(err, IsNil)
2325
266
	defer lock2.Unlock()
2326
267
2327
268
	// The expired lock was forcefully killed. Unlocking it must
2328
269
	// not interfere with lock2 which is still alive.
2329
270
	lock1.Unlock()
2330
271
2331
272
	// The above statement was a NOOP and lock2 is still in effect,
2332
273
	// so attempting another lock must necessarily fail.
2333
274
	lock3, err := s.store.LockUpdates(urls)
2334
275
	c.Check(err == store.ErrUpdateConflict, Equals, true)
2335
276
	c.Check(lock3, IsNil)
2336
277
}
2337
278
2338
279
func (s *StoreSuite) TestConflictingUpdate(c *C) {
2339
280
	// This test checks that if for whatever reason the locking
2340
281
	// safety-net fails, adding two charms in parallel still
2341
282
	// results in a sane outcome.
2342
283
	url := charm.MustParseURL("cs:oneiric/wordpress")
2343
284
	urls := []*charm.URL{url}
2344
285
2345
286
	pub1, err := s.store.CharmPublisher(urls, "some-digest")
2346
287
	c.Assert(err, IsNil)
2347
288
	c.Assert(pub1.Revision(), Equals, 0)
2348
289
2349
290
	pub2, err := s.store.CharmPublisher(urls, "some-digest")
2350
291
	c.Assert(err, IsNil)
2351
292
	c.Assert(pub2.Revision(), Equals, 0)
2352
293
2353
294
	// The first publishing attempt should work.
2354
295
	err = pub2.Publish(&FakeCharmDir{})
2355
296
	c.Assert(err, IsNil)
2356
297
2357
298
	// Attempting to finish the second attempt should break,
2358
299
	// since it lost the race and the given revision is already
2359
300
	// in place.
2360
301
	err = pub1.Publish(&FakeCharmDir{})
2361
302
	c.Assert(err, Equals, store.ErrUpdateConflict)
2362
303
}
2363
304
2364
305
func (s *StoreSuite) TestRedundantUpdate(c *C) {
2365
306
	urlA := charm.MustParseURL("cs:oneiric/wordpress-a")
2366
307
	urlB := charm.MustParseURL("cs:oneiric/wordpress-b")
2367
308
	urls := []*charm.URL{urlA, urlB}
2368
309
2369
310
	pub, err := s.store.CharmPublisher(urls, "digest-0")
2370
311
	c.Assert(err, IsNil)
2371
312
	c.Assert(pub.Revision(), Equals, 0)
2372
313
	err = pub.Publish(&FakeCharmDir{})
2373
314
	c.Assert(err, IsNil)
2374
315
2375
316
	// All charms are already on digest-0.
2376
317
	pub, err = s.store.CharmPublisher(urls, "digest-0")
2377
318
	c.Assert(err, ErrorMatches, "charm is up-to-date")
2378
319
	c.Assert(err, Equals, store.ErrRedundantUpdate)
2379
320
	c.Assert(pub, IsNil)
2380
321
2381
322
	// Now add a second revision just for wordpress-b.
2382
323
	pub, err = s.store.CharmPublisher(urls[1:], "digest-1")
2383
324
	c.Assert(err, IsNil)
2384
325
	c.Assert(pub.Revision(), Equals, 1)
2385
326
	err = pub.Publish(&FakeCharmDir{})
2386
327
	c.Assert(err, IsNil)
2387
328
2388
329
	// Same digest bumps revision because one of them was old.
2389
330
	pub, err = s.store.CharmPublisher(urls, "digest-1")
2390
331
	c.Assert(err, IsNil)
2391
332
	c.Assert(pub.Revision(), Equals, 2)
2392
333
	err = pub.Publish(&FakeCharmDir{})
2393
334
	c.Assert(err, IsNil)
2394
335
}
2395
336
2396
337
const fakeRevZeroSha = "319095521ac8a62fa1e8423351973512ecca8928c9f62025e37de57c9ef07a53"
2397
338
2398
339
func (s *StoreSuite) TestCharmBundleData(c *C) {
2399
340
	url := charm.MustParseURL("cs:oneiric/wordpress")
2400
341
	urls := []*charm.URL{url}
2401
342
2402
343
	pub, err := s.store.CharmPublisher(urls, "key")
2403
344
	c.Assert(err, IsNil)
2404
345
	c.Assert(pub.Revision(), Equals, 0)
2405
346
2406
347
	err = pub.Publish(&FakeCharmDir{})
2407
348
	c.Assert(err, IsNil)
2408
349
2409
350
	info, rc, err := s.store.OpenCharm(url)
2410
351
	c.Assert(err, IsNil)
2411
352
	c.Check(info.BundleSha256(), Equals, fakeRevZeroSha)
2412
353
	c.Check(info.BundleSize(), Equals, int64(len("charm-revision-0")))
2413
354
	err = rc.Close()
2414
355
	c.Check(err, IsNil)
2415
356
}
2416
357
2417
358
func (s *StoreSuite) TestLogCharmEventWithRevisionedURL(c *C) {
2418
359
	url := charm.MustParseURL("cs:oneiric/wordpress-0")
2419
360
	event := &store.CharmEvent{
2420
361
		Kind:   store.EventPublishError,
2421
362
		Digest: "some-digest",
2422
363
		URLs:   []*charm.URL{url},
2423
364
	}
2424
365
	err := s.store.LogCharmEvent(event)
2425
366
	c.Assert(err, ErrorMatches, "LogCharmEvent: got charm URL with revision: cs:oneiric/wordpress-0")
2426
367
2427
368
	// This may work in the future, but not now.
2428
369
	event, err = s.store.CharmEvent(url, "some-digest")
2429
370
	c.Assert(err, ErrorMatches, "CharmEvent: got charm URL with revision: cs:oneiric/wordpress-0")
2430
371
	c.Assert(event, IsNil)
2431
372
}
2432
373
2433
374
func (s *StoreSuite) TestLogCharmEvent(c *C) {
2434
375
	url1 := charm.MustParseURL("cs:oneiric/wordpress")
2435
376
	url2 := charm.MustParseURL("cs:oneiric/mysql")
2436
377
	urls := []*charm.URL{url1, url2}
2437
378
2438
379
	event1 := &store.CharmEvent{
2439
380
		Kind:     store.EventPublished,
2440
381
		Revision: 42,
2441
382
		Digest:   "revKey1",
2442
383
		URLs:     urls,
2443
384
		Warnings: []string{"A warning."},
2444
385
		Time:     time.Unix(1, 0),
2445
386
	}
2446
387
	event2 := &store.CharmEvent{
2447
388
		Kind:     store.EventPublished,
2448
389
		Revision: 42,
2449
390
		Digest:   "revKey2",
2450
391
		URLs:     urls,
2451
392
		Time:     time.Unix(1, 0),
2452
393
	}
2453
394
	event3 := &store.CharmEvent{
2454
395
		Kind:   store.EventPublishError,
2455
396
		Digest: "revKey2",
2456
397
		Errors: []string{"An error."},
2457
398
		URLs:   urls[:1],
2458
399
	}
2459
400
2460
401
	for _, event := range []*store.CharmEvent{event1, event2, event3} {
2461
402
		err := s.store.LogCharmEvent(event)
2462
403
		c.Assert(err, IsNil)
2463
404
	}
2464
405
2465
406
	events := s.Session.DB("juju").C("events")
2466
407
	var s1, s2 map[string]interface{}
2467
408
2468
409
	err := events.Find(bson.M{"digest": "revKey1"}).One(&s1)
2469
410
	c.Assert(err, IsNil)
2470
411
	c.Assert(s1["kind"], Equals, int(store.EventPublished))
2471
412
	c.Assert(s1["urls"], DeepEquals, []interface{}{"cs:oneiric/wordpress", "cs:oneiric/mysql"})
2472
413
	c.Assert(s1["warnings"], DeepEquals, []interface{}{"A warning."})
2473
414
	c.Assert(s1["errors"], IsNil)
2474
415
	c.Assert(s1["time"], DeepEquals, time.Unix(1, 0))
2475
416
2476
417
	err = events.Find(bson.M{"digest": "revKey2", "kind": store.EventPublishError}).One(&s2)
2477
418
	c.Assert(err, IsNil)
2478
419
	c.Assert(s2["urls"], DeepEquals, []interface{}{"cs:oneiric/wordpress"})
2479
420
	c.Assert(s2["warnings"], IsNil)
2480
421
	c.Assert(s2["errors"], DeepEquals, []interface{}{"An error."})
2481
422
	c.Assert(s2["time"].(time.Time).After(bson.Now().Add(-10e9)), Equals, true)
2482
423
2483
424
	// Mongo stores timestamps in milliseconds, so chop
2484
425
	// off the extra bits for comparison.
2485
426
	event3.Time = time.Unix(0, event3.Time.UnixNano()/1e6*1e6)
2486
427
2487
428
	event, err := s.store.CharmEvent(urls[0], "revKey2")
2488
429
	c.Assert(err, IsNil)
2489
430
	c.Assert(event, DeepEquals, event3)
2490
431
2491
432
	event, err = s.store.CharmEvent(urls[1], "revKey1")
2492
433
	c.Assert(err, IsNil)
2493
434
	c.Assert(event, DeepEquals, event1)
2494
435
2495
436
	event, err = s.store.CharmEvent(urls[1], "revKeyX")
2496
437
	c.Assert(err, Equals, store.ErrNotFound)
2497
438
	c.Assert(event, IsNil)
2498
439
}
2499
440
2500
441
func (s *StoreSuite) TestCounters(c *C) {
2501
442
	sum, err := s.store.SumCounter([]string{"a"}, false)
2502
443
	c.Assert(err, IsNil)
2503
444
	c.Assert(sum, Equals, int64(0))
2504
445
2505
446
	for i := 0; i < 10; i++ {
2506
447
		err := s.store.IncCounter([]string{"a", "b", "c"})
2507
448
		c.Assert(err, IsNil)
2508
449
	}
2509
450
	for i := 0; i < 7; i++ {
2510
451
		s.store.IncCounter([]string{"a", "b"})
2511
452
		c.Assert(err, IsNil)
2512
453
	}
2513
454
	for i := 0; i < 3; i++ {
2514
455
		s.store.IncCounter([]string{"a", "z", "b"})
2515
456
		c.Assert(err, IsNil)
2516
457
	}
2517
458
2518
459
	tests := []struct {
2519
460
		key    []string
2520
461
		prefix bool
2521
462
		result int64
2522
463
	}{
2523
464
		{[]string{"a", "b", "c"}, false, 10},
2524
465
		{[]string{"a", "b"}, false, 7},
2525
466
		{[]string{"a", "z", "b"}, false, 3},
2526
467
		{[]string{"a", "b", "c"}, true, 10},
2527
468
		{[]string{"a", "b"}, true, 17},
2528
469
		{[]string{"a"}, true, 20},
2529
470
		{[]string{"b"}, true, 0},
2530
471
	}
2531
472
2532
473
	for _, t := range tests {
2533
474
		c.Logf("Test: %#v\n", t)
2534
475
		sum, err := s.store.SumCounter(t.key, t.prefix)
2535
476
		c.Assert(err, IsNil)
2536
477
		c.Assert(sum, Equals, t.result)
2537
478
	}
2538
479
2539
480
	// High-level interface works. Now check that the data is
2540
481
	// stored correctly.
2541
482
	counters := s.Session.DB("juju").C("stat.counters")
2542
483
	docs1, err := counters.Count()
2543
484
	c.Assert(err, IsNil)
2544
485
	if docs1 != 3 && docs1 != 4 {
2545
486
		fmt.Errorf("Expected 3 or 4 docs in counters collection, got %d", docs1)
2546
487
	}
2547
488
2548
489
	// Hack times so that the next operation adds another document.
2549
490
	err = counters.Update(nil, bson.D{{"$set", bson.D{{"t", 1}}}})
2550
491
	c.Check(err, IsNil)
2551
492
2552
493
	err = s.store.IncCounter([]string{"a", "b", "c"})
2553
494
	c.Assert(err, IsNil)
2554
495
2555
496
	docs2, err := counters.Count()
2556
497
	c.Assert(err, IsNil)
2557
498
	c.Assert(docs2, Equals, docs1+1)
2558
499
2559
500
	sum, err = s.store.SumCounter([]string{"a", "b", "c"}, false)
2560
501
	c.Assert(err, IsNil)
2561
502
	c.Assert(sum, Equals, int64(11))
2562
503
2563
504
	sum, err = s.store.SumCounter([]string{"a"}, true)
2564
505
	c.Assert(err, IsNil)
2565
506
	c.Assert(sum, Equals, int64(21))
2566
507
}
2567
508
2568
509
func (s *StoreSuite) TestCountersReadOnlySum(c *C) {
2569
510
	// Summing up an unknown key shouldn't add the key to the database.
2570
511
	sum, err := s.store.SumCounter([]string{"a", "b", "c"}, false)
2571
512
	c.Assert(err, IsNil)
2572
513
	c.Assert(sum, Equals, int64(0))
2573
514
2574
515
	tokens := s.Session.DB("juju").C("stat.tokens")
2575
516
	n, err := tokens.Count()
2576
517
	c.Assert(err, IsNil)
2577
518
	c.Assert(n, Equals, 0)
2578
519
}
2579
520
2580
521
func (s *StoreSuite) TestCountersTokenCaching(c *C) {
2581
522
	sum, err := s.store.SumCounter([]string{"a"}, false)
2582
523
	c.Assert(err, IsNil)
2583
524
	c.Assert(sum, Equals, int64(0))
2584
525
2585
526
	const genSize = 512
2586
527
2587
528
	// All of these will be cached, as we have two generations
2588
529
	// of genSize entries each.
2589
530
	for i := 0; i < genSize*2; i++ {
2590
531
		err := s.store.IncCounter([]string{strconv.Itoa(i)})
2591
532
		c.Assert(err, IsNil)
2592
533
	}
2593
534
2594
535
	// Now go behind the scenes and corrupt all the tokens.
2595
536
	tokens := s.Session.DB("juju").C("stat.tokens")
2596
537
	iter := tokens.Find(nil).Iter()
2597
538
	var t struct {
2598
539
		Id    int    "_id"
2599
540
		Token string "t"
2600
541
	}
2601
542
	for iter.Next(&t) {
2602
543
		err := tokens.UpdateId(t.Id, bson.M{"$set": bson.M{"t": "corrupted" + t.Token}})
2603
544
		c.Assert(err, IsNil)
2604
545
	}
2605
546
	c.Assert(iter.Err(), IsNil)
2606
547
2607
548
	// We can consult the counters for the cached entries still.
2608
549
	// First, check that the newest generation is good.
2609
550
	for i := genSize; i < genSize*2; i++ {
2610
551
		n, err := s.store.SumCounter([]string{strconv.Itoa(i)}, false)
2611
552
		c.Assert(err, IsNil)
2612
553
		c.Assert(n, Equals, int64(1))
2613
554
	}
2614
555
2615
556
	// Now, we can still access a single entry of the older generation,
2616
557
	// but this will cause the generations to flip and thus the rest
2617
558
	// of the old generation will go away as the top half of the
2618
559
	// entries is turned into the old generation.
2619
560
	n, err := s.store.SumCounter([]string{"0"}, false)
2620
561
	c.Assert(err, IsNil)
2621
562
	c.Assert(n, Equals, int64(1))
2622
563
2623
564
	// Now we've lost access to the rest of the old generation.
2624
565
	for i := 1; i < genSize; i++ {
2625
566
		n, err := s.store.SumCounter([]string{strconv.Itoa(i)}, false)
2626
567
		c.Assert(err, IsNil)
2627
568
		c.Assert(n, Equals, int64(0))
2628
569
	}
2629
570
2630
571
	// But we still have all of the top half available since it was
2631
572
	// moved into the old generation.
2632
573
	for i := genSize; i < genSize*2; i++ {
2633
574
		n, err := s.store.SumCounter([]string{strconv.Itoa(i)}, false)
2634
575
		c.Assert(err, IsNil)
2635
576
		c.Assert(n, Equals, int64(1))
2636
577
	}
2637
578
}
2638
579
2639
580
func (s *StoreSuite) TestCounterTokenUniqueness(c *C) {
2640
581
	var wg0, wg1 sync.WaitGroup
2641
582
	wg0.Add(10)
2642
583
	wg1.Add(10)
2643
584
	for i := 0; i < 10; i++ {
2644
585
		go func() {
2645
586
			wg0.Done()
2646
587
			wg0.Wait()
2647
588
			defer wg1.Done()
2648
589
			err := s.store.IncCounter([]string{"a"})
2649
590
			c.Check(err, IsNil)
2650
591
		}()
2651
592
	}
2652
593
	wg1.Wait()
2653
594
2654
595
	sum, err := s.store.SumCounter([]string{"a"}, false)
2655
596
	c.Assert(err, IsNil)
2656
597
	c.Assert(sum, Equals, int64(10))
2657
598
}
2658
599
2659
600
func (s *TrivialSuite) TestEventString(c *C) {
2660
601
	c.Assert(store.EventPublished, Matches, "published")
2661
602
	c.Assert(store.EventPublishError, Matches, "publish-error")
2662
603
	for kind := store.CharmEventKind(1); kind < store.EventKindCount; kind++ {
2663
604
		// This guarantees the switch in String is properly
2664
605
		// updated with new event kinds.
2665
606
		c.Assert(kind.String(), Matches, "[a-z-]+")
2666
607
	}
2667
608
}
Status:	Merged
Approved by:	William Reade on 2013-02-04
Approved revision:	no longer in the source branch.
Merged at revision:	886
Proposed branch:	lp:~allenap/juju-core/break-out-juju-store
Merge into:	lp:~juju/juju-core/trunk
Diff against target:	2667 lines (+0/-2600) 13 files modified cmd/charmd/config.yaml (+0/-2) cmd/charmd/main.go (+0/-74) cmd/charmload/config.yaml (+0/-1) cmd/charmload/main.go (+0/-75) store/branch.go (+0/-152) store/branch_test.go (+0/-238) store/lpad.go (+0/-113) store/lpad_test.go (+0/-68) store/mgo_test.go (+0/-95) store/server.go (+0/-191) store/server_test.go (+0/-209) store/store.go (+0/-774) store/store_test.go (+0/-608)
To merge this branch:	bzr merge lp:~allenap/juju-core/break-out-juju-store
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
The Go Language Gophers		2013-01-09	Pending
Review via email: mp+142564@code.launchpad.net