1
=== modified file 'cmd/plugins/juju-metadata/imagemetadata.go'
2
--- cmd/plugins/juju-metadata/imagemetadata.go	2013-10-16 03:31:36 +0000
3
+++ cmd/plugins/juju-metadata/imagemetadata.go	2013-10-17 05:10:43 +0000
4
@@ -17,6 +17,7 @@
5
17
	"launchpad.net/juju-core/environs/filestorage"
17
	"launchpad.net/juju-core/environs/filestorage"
6
18
	"launchpad.net/juju-core/environs/imagemetadata"
18
	"launchpad.net/juju-core/environs/imagemetadata"
7
19
	"launchpad.net/juju-core/environs/simplestreams"
19
	"launchpad.net/juju-core/environs/simplestreams"
8
20
	"launchpad.net/juju-core/utils"
9
20
)
21
)
10
21
22
11
22
// ImageMetadataCommand is used to write out simplestreams image metadata information.
23
// ImageMetadataCommand is used to write out simplestreams image metadata information.
12
@@ -125,17 +126,17 @@
13
125
126
14
126
var helpDoc = `
127
var helpDoc = `
15
127
image metadata files have been written to:
128
image metadata files have been written to:
17
128
%q.
129
%s.
18
129
For Juju to use this metadata, the files need to be put into the
130
For Juju to use this metadata, the files need to be put into the
19
130
image metadata search path. There are 2 options:
131
image metadata search path. There are 2 options:
20
131
132
21
132
1. Use tools-url in $JUJU_HOME/environments.yaml
133
1. Use tools-url in $JUJU_HOME/environments.yaml
22
133
Configure a http server to serve the contents of
134
Configure a http server to serve the contents of
24
134
%q
135
%s
25
135
and set the value of tools-url accordingly.
136
and set the value of tools-url accordingly.
26
136
137
27
137
2. Upload the contents of
138
2. Upload the contents of
29
138
%q
139
%s
30
139
to your cloud's private storage (for ec2 and openstack).
140
to your cloud's private storage (for ec2 and openstack).
31
140
eg for openstack
141
eg for openstack
32
141
"cd %s; swift upload %s streams/v1/*"
142
"cd %s; swift upload %s streams/v1/*"
33
@@ -145,7 +146,7 @@
34
145
func (c *ImageMetadataCommand) Run(context *cmd.Context) error {
146
func (c *ImageMetadataCommand) Run(context *cmd.Context) error {
35
146
	out := context.Stdout
147
	out := context.Stdout
36
147
148
38
148
	im := imagemetadata.ImageMetadata{
149
	im := &imagemetadata.ImageMetadata{
39
149
		Id:   c.ImageId,
150
		Id:   c.ImageId,
40
150
		Arch: c.Arch,
151
		Arch: c.Arch,
41
151
	}
152
	}
42
@@ -157,11 +158,12 @@
43
157
	if err != nil {
158
	if err != nil {
44
158
		return err
159
		return err
45
159
	}
160
	}
47
160
	err = imagemetadata.WriteMetadata(c.Series, &im, &cloudSpec, targetStorage)
161
	err = imagemetadata.MergeAndWriteMetadata(c.Series, []*imagemetadata.ImageMetadata{im}, &cloudSpec, targetStorage)
48
161
	if err != nil {
162
	if err != nil {
49
162
		return fmt.Errorf("image metadata files could not be created: %v", err)
163
		return fmt.Errorf("image metadata files could not be created: %v", err)
50
163
	}
164
	}
51
164
	dest := filepath.Join(c.Dir, "streams", "v1")
165
	dest := filepath.Join(c.Dir, "streams", "v1")
53
165
	fmt.Fprintf(out, fmt.Sprintf(helpDoc, dest, c.Dir, c.Dir, c.Dir, c.privateStorage))
166
	dir := utils.NormalizePath(c.Dir)
54
167
	fmt.Fprintf(out, fmt.Sprintf(helpDoc, dest, dir, dir, dir, c.privateStorage))
55
166
	return nil
168
	return nil
56
167
}
169
}
57
168
170
58
=== modified file 'cmd/plugins/juju-metadata/imagemetadata_test.go'
59
--- cmd/plugins/juju-metadata/imagemetadata_test.go	2013-10-16 03:31:36 +0000
60
+++ cmd/plugins/juju-metadata/imagemetadata_test.go	2013-10-17 05:10:43 +0000
61
@@ -4,7 +4,6 @@
62
4
package main
4
package main
63
5
5
64
6
import (
6
import (
65
7
	"bytes"
66
8
	"encoding/json"
7
	"encoding/json"
67
9
	"fmt"
8
	"fmt"
68
10
	"io/ioutil"
9
	"io/ioutil"
69
@@ -89,10 +88,10 @@
70
89
	c.Assert(err, gc.IsNil)
88
	c.Assert(err, gc.IsNil)
71
90
	c.Assert(indices.(map[string]interface{})["format"], gc.Equals, "index:1.0")
89
	c.Assert(indices.(map[string]interface{})["format"], gc.Equals, "index:1.0")
72
91
	prodId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", seriesVersions[expected.series], expected.arch)
90
	prodId := fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", seriesVersions[expected.series], expected.arch)
77
92
	c.Assert(strings.Contains(content, prodId), jc.IsTrue)
91
	c.Assert(content, jc.Contains, prodId)
78
93
	c.Assert(strings.Contains(content, fmt.Sprintf(`"region": %q`, expected.region)), jc.IsTrue)
92
	c.Assert(content, jc.Contains, fmt.Sprintf(`"region": %q`, expected.region))
79
94
	c.Assert(strings.Contains(content, fmt.Sprintf(`"endpoint": %q`, expected.endpoint)), jc.IsTrue)
93
	c.Assert(content, jc.Contains, fmt.Sprintf(`"endpoint": %q`, expected.endpoint))
80
95
	c.Assert(strings.Contains(content, fmt.Sprintf(`"path": "streams/v1/%s"`, imageFileName)), jc.IsTrue)
94
	c.Assert(content, jc.Contains, fmt.Sprintf(`"path": "streams/v1/%s"`, imageFileName))
81
96
95
82
97
	imagepath := filepath.Join(s.dir, "streams", "v1", imageFileName)
96
	imagepath := filepath.Join(s.dir, "streams", "v1", imageFileName)
83
98
	data, err = ioutil.ReadFile(imagepath)
97
	data, err = ioutil.ReadFile(imagepath)
84
@@ -102,8 +101,8 @@
85
102
	err = json.Unmarshal(data, &images)
101
	err = json.Unmarshal(data, &images)
86
103
	c.Assert(err, gc.IsNil)
102
	c.Assert(err, gc.IsNil)
87
104
	c.Assert(images.(map[string]interface{})["format"], gc.Equals, "products:1.0")
103
	c.Assert(images.(map[string]interface{})["format"], gc.Equals, "products:1.0")
90
105
	c.Assert(strings.Contains(content, prodId), gc.Equals, true)
104
	c.Assert(content, jc.Contains, prodId)
91
106
	c.Assert(strings.Contains(content, `"id": "1234"`), gc.Equals, true)
105
	c.Assert(content, jc.Contains, `"id": "1234"`)
92
107
}
106
}
93
108
107
94
109
const (
108
const (
95
@@ -117,12 +116,12 @@
96
117
		&ImageMetadataCommand{}, ctx, []string{
116
		&ImageMetadataCommand{}, ctx, []string{
97
118
			"-d", s.dir, "-i", "1234", "-r", "region", "-a", "arch", "-u", "endpoint", "-s", "raring"})
117
			"-d", s.dir, "-i", "1234", "-r", "region", "-a", "arch", "-u", "endpoint", "-s", "raring"})
98
119
	c.Assert(code, gc.Equals, 0)
118
	c.Assert(code, gc.Equals, 0)
100
120
	errOut := ctx.Stdout.(*bytes.Buffer).String()
119
	out := testing.Stdout(ctx)
101
121
	expected := expectedMetadata{
120
	expected := expectedMetadata{
102
122
		series: "raring",
121
		series: "raring",
103
123
		arch:   "arch",
122
		arch:   "arch",
104
124
	}
123
	}
106
125
	s.assertCommandOutput(c, expected, errOut, defaultIndexFileName, defaultImageFileName)
124
	s.assertCommandOutput(c, expected, out, defaultIndexFileName, defaultImageFileName)
107
126
}
125
}
108
127
126
109
128
func (s *ImageMetadataSuite) TestImageMetadataFilesDefaultArch(c *gc.C) {
127
func (s *ImageMetadataSuite) TestImageMetadataFilesDefaultArch(c *gc.C) {
110
@@ -131,12 +130,12 @@
111
131
		&ImageMetadataCommand{}, ctx, []string{
130
		&ImageMetadataCommand{}, ctx, []string{
112
132
			"-d", s.dir, "-i", "1234", "-r", "region", "-u", "endpoint", "-s", "raring"})
131
			"-d", s.dir, "-i", "1234", "-r", "region", "-u", "endpoint", "-s", "raring"})
113
133
	c.Assert(code, gc.Equals, 0)
132
	c.Assert(code, gc.Equals, 0)
115
134
	errOut := ctx.Stdout.(*bytes.Buffer).String()
133
	out := testing.Stdout(ctx)
116
135
	expected := expectedMetadata{
134
	expected := expectedMetadata{
117
136
		series: "raring",
135
		series: "raring",
118
137
		arch:   "amd64",
136
		arch:   "amd64",
119
138
	}
137
	}
121
139
	s.assertCommandOutput(c, expected, errOut, defaultIndexFileName, defaultImageFileName)
138
	s.assertCommandOutput(c, expected, out, defaultIndexFileName, defaultImageFileName)
122
140
}
139
}
123
141
140
124
142
func (s *ImageMetadataSuite) TestImageMetadataFilesDefaultSeries(c *gc.C) {
141
func (s *ImageMetadataSuite) TestImageMetadataFilesDefaultSeries(c *gc.C) {
125
@@ -145,12 +144,12 @@
126
145
		&ImageMetadataCommand{}, ctx, []string{
144
		&ImageMetadataCommand{}, ctx, []string{
127
146
			"-d", s.dir, "-i", "1234", "-r", "region", "-a", "arch", "-u", "endpoint"})
145
			"-d", s.dir, "-i", "1234", "-r", "region", "-a", "arch", "-u", "endpoint"})
128
147
	c.Assert(code, gc.Equals, 0)
146
	c.Assert(code, gc.Equals, 0)
130
148
	errOut := ctx.Stdout.(*bytes.Buffer).String()
147
	out := testing.Stdout(ctx)
131
149
	expected := expectedMetadata{
148
	expected := expectedMetadata{
132
150
		series: "precise",
149
		series: "precise",
133
151
		arch:   "arch",
150
		arch:   "arch",
134
152
	}
151
	}
136
153
	s.assertCommandOutput(c, expected, errOut, defaultIndexFileName, defaultImageFileName)
152
	s.assertCommandOutput(c, expected, out, defaultIndexFileName, defaultImageFileName)
137
154
}
153
}
138
155
154
139
156
func (s *ImageMetadataSuite) TestImageMetadataFilesUsingEnv(c *gc.C) {
155
func (s *ImageMetadataSuite) TestImageMetadataFilesUsingEnv(c *gc.C) {
140
@@ -158,14 +157,14 @@
141
158
	code := cmd.Main(
157
	code := cmd.Main(
142
159
		&ImageMetadataCommand{}, ctx, []string{"-d", s.dir, "-e", "ec2", "-i", "1234"})
158
		&ImageMetadataCommand{}, ctx, []string{"-d", s.dir, "-e", "ec2", "-i", "1234"})
143
160
	c.Assert(code, gc.Equals, 0)
159
	c.Assert(code, gc.Equals, 0)
145
161
	errOut := ctx.Stdout.(*bytes.Buffer).String()
160
	out := testing.Stdout(ctx)
146
162
	expected := expectedMetadata{
161
	expected := expectedMetadata{
147
163
		series:   "precise",
162
		series:   "precise",
148
164
		arch:     "amd64",
163
		arch:     "amd64",
149
165
		region:   "us-east-1",
164
		region:   "us-east-1",
150
166
		endpoint: "https://ec2.us-east-1.amazonaws.com",
165
		endpoint: "https://ec2.us-east-1.amazonaws.com",
151
167
	}
166
	}
153
168
	s.assertCommandOutput(c, expected, errOut, defaultIndexFileName, defaultImageFileName)
167
	s.assertCommandOutput(c, expected, out, defaultIndexFileName, defaultImageFileName)
154
169
}
168
}
155
170
169
156
171
func (s *ImageMetadataSuite) TestImageMetadataFilesUsingEnvWithRegionOverride(c *gc.C) {
170
func (s *ImageMetadataSuite) TestImageMetadataFilesUsingEnvWithRegionOverride(c *gc.C) {
157
@@ -174,14 +173,14 @@
158
174
		&ImageMetadataCommand{}, ctx, []string{
173
		&ImageMetadataCommand{}, ctx, []string{
159
175
			"-d", s.dir, "-e", "ec2", "-r", "us-west-1", "-u", "https://ec2.us-west-1.amazonaws.com", "-i", "1234"})
174
			"-d", s.dir, "-e", "ec2", "-r", "us-west-1", "-u", "https://ec2.us-west-1.amazonaws.com", "-i", "1234"})
160
176
	c.Assert(code, gc.Equals, 0)
175
	c.Assert(code, gc.Equals, 0)
162
177
	errOut := ctx.Stdout.(*bytes.Buffer).String()
176
	out := testing.Stdout(ctx)
163
178
	expected := expectedMetadata{
177
	expected := expectedMetadata{
164
179
		series:   "precise",
178
		series:   "precise",
165
180
		arch:     "amd64",
179
		arch:     "amd64",
166
181
		region:   "us-west-1",
180
		region:   "us-west-1",
167
182
		endpoint: "https://ec2.us-west-1.amazonaws.com",
181
		endpoint: "https://ec2.us-west-1.amazonaws.com",
168
183
	}
182
	}
170
184
	s.assertCommandOutput(c, expected, errOut, defaultIndexFileName, defaultImageFileName)
183
	s.assertCommandOutput(c, expected, out, defaultIndexFileName, defaultImageFileName)
171
185
}
184
}
172
186
185
173
187
func (s *ImageMetadataSuite) TestImageMetadataFilesUsingEnvWithNoHasRegion(c *gc.C) {
186
func (s *ImageMetadataSuite) TestImageMetadataFilesUsingEnvWithNoHasRegion(c *gc.C) {
174
@@ -190,14 +189,14 @@
175
190
		&ImageMetadataCommand{}, ctx, []string{
189
		&ImageMetadataCommand{}, ctx, []string{
176
191
			"-d", s.dir, "-e", "azure", "-r", "region", "-u", "endpoint", "-i", "1234"})
190
			"-d", s.dir, "-e", "azure", "-r", "region", "-u", "endpoint", "-i", "1234"})
177
192
	c.Assert(code, gc.Equals, 0)
191
	c.Assert(code, gc.Equals, 0)
179
193
	errOut := ctx.Stdout.(*bytes.Buffer).String()
192
	out := testing.Stdout(ctx)
180
194
	expected := expectedMetadata{
193
	expected := expectedMetadata{
181
195
		series:   "raring",
194
		series:   "raring",
182
196
		arch:     "amd64",
195
		arch:     "amd64",
183
197
		region:   "region",
196
		region:   "region",
184
198
		endpoint: "endpoint",
197
		endpoint: "endpoint",
185
199
	}
198
	}
187
200
	s.assertCommandOutput(c, expected, errOut, defaultIndexFileName, defaultImageFileName)
199
	s.assertCommandOutput(c, expected, out, defaultIndexFileName, defaultImageFileName)
188
201
}
200
}
189
202
201
190
203
type errTestParams struct {
202
type errTestParams struct {
191
204
203
192
=== modified file 'cmd/plugins/juju-metadata/validateimagemetadata_test.go'
193
--- cmd/plugins/juju-metadata/validateimagemetadata_test.go	2013-10-17 04:53:59 +0000
194
+++ cmd/plugins/juju-metadata/validateimagemetadata_test.go	2013-10-17 05:10:43 +0000
195
@@ -66,7 +66,7 @@
196
66
}
66
}
197
67
67
198
68
func (s *ValidateImageMetadataSuite) makeLocalMetadata(c *gc.C, id, region, series, endpoint string) error {
68
func (s *ValidateImageMetadataSuite) makeLocalMetadata(c *gc.C, id, region, series, endpoint string) error {
200
69
	im := imagemetadata.ImageMetadata{
69
	im := &imagemetadata.ImageMetadata{
201
70
		Id:   id,
70
		Id:   id,
202
71
		Arch: "amd64",
71
		Arch: "amd64",
203
72
	}
72
	}
204
@@ -78,7 +78,7 @@
205
78
	if err != nil {
78
	if err != nil {
206
79
		return err
79
		return err
207
80
	}
80
	}
209
81
	err = imagemetadata.WriteMetadata(series, &im, &cloudSpec, targetStorage)
81
	err = imagemetadata.MergeAndWriteMetadata(series, []*imagemetadata.ImageMetadata{im}, &cloudSpec, targetStorage)
210
82
	if err != nil {
82
	if err != nil {
211
83
		return err
83
		return err
212
84
	}
84
	}
213
@@ -105,10 +105,8 @@
214
105
	s.LoggingSuite.SetUpTest(c)
105
	s.LoggingSuite.SetUpTest(c)
215
106
	s.metadataDir = c.MkDir()
106
	s.metadataDir = c.MkDir()
216
107
	s.home = coretesting.MakeFakeHome(c, metadataTestEnvConfig)
107
	s.home = coretesting.MakeFakeHome(c, metadataTestEnvConfig)
221
108
	restore := testbase.PatchEnvironment("AWS_ACCESS_KEY_ID", "access")
108
	s.PatchEnvironment("AWS_ACCESS_KEY_ID", "access")
222
109
	s.AddCleanup(func(*gc.C) { restore() })
109
	s.PatchEnvironment("AWS_SECRET_ACCESS_KEY", "secret")
219
110
	restore = testbase.PatchEnvironment("AWS_SECRET_ACCESS_KEY", "secret")
220
111
	s.AddCleanup(func(*gc.C) { restore() })
223
112
}
110
}
224
113
111
225
114
func (s *ValidateImageMetadataSuite) TearDownTest(c *gc.C) {
112
func (s *ValidateImageMetadataSuite) TearDownTest(c *gc.C) {
226
115
113
227
=== modified file 'environs/imagemetadata/generate.go'
228
--- environs/imagemetadata/generate.go	2013-10-11 06:44:35 +0000
229
+++ environs/imagemetadata/generate.go	2013-10-17 05:10:43 +0000
230
@@ -5,16 +5,96 @@
231
5
5
232
6
import (
6
import (
233
7
	"bytes"
7
	"bytes"
234
8
	"fmt"
235
8
	"time"
9
	"time"
236
9
10
237
10
	"launchpad.net/juju-core/environs/simplestreams"
11
	"launchpad.net/juju-core/environs/simplestreams"
238
11
	"launchpad.net/juju-core/environs/storage"
12
	"launchpad.net/juju-core/environs/storage"
239
13
	"launchpad.net/juju-core/errors"
240
12
)
14
)
241
13
15
246
14
func WriteMetadata(series string, im *ImageMetadata, cloudSpec *simplestreams.CloudSpec, metadataStore storage.Storage) error {
16
// MergeAndWriteMetadata reads the existing metadata from storage (if any),
247
15
	metadataInfo, err := generateMetadata(series, im, cloudSpec)
17
// and merges it with supplied metadata, writing the resulting metadata is written to storage.
248
16
	if err != nil {
18
func MergeAndWriteMetadata(series string, metadata []*ImageMetadata, cloudSpec *simplestreams.CloudSpec,
249
17
		return err
19
	metadataStore storage.Storage) error {
250
20
251
21
	existingMetadata, err := readMetadata(metadataStore)
252
22
	if err != nil {
253
23
		return err
254
24
	}
255
25
	seriesVersion, err := simplestreams.SeriesVersion(series)
256
26
	if err != nil {
257
27
		return err
258
28
	}
259
29
	toWrite, allCloudSpec := mergeMetadata(seriesVersion, cloudSpec, metadata, existingMetadata)
260
30
	return writeMetadata(toWrite, allCloudSpec, metadataStore)
261
31
}
262
32
263
33
// readMetadata reads the image metadata from metadataStore.
264
34
func readMetadata(metadataStore storage.Storage) ([]*ImageMetadata, error) {
265
35
	// Read any existing metadata so we can merge the new tools metadata with what's there.
266
36
	dataSource := storage.NewStorageSimpleStreamsDataSource(metadataStore, "")
267
37
	imageConstraint := NewImageConstraint(simplestreams.LookupParams{})
268
38
	existingMetadata, err := Fetch(
269
39
		[]simplestreams.DataSource{dataSource}, simplestreams.DefaultIndexPath, imageConstraint, false)
270
40
	if err != nil && !errors.IsNotFoundError(err) {
271
41
		return nil, err
272
42
	}
273
43
	return existingMetadata, nil
274
44
}
275
45
276
46
func mapKey(im *ImageMetadata) string {
277
47
	return fmt.Sprintf("%s-%s", im.productId(), im.RegionName)
278
48
}
279
49
280
50
// mergeMetadata merges the newMetadata into existingMetadata, overwriting existing matching image records.
281
51
func mergeMetadata(seriesVersion string, cloudSpec *simplestreams.CloudSpec, newMetadata,
282
52
	existingMetadata []*ImageMetadata) ([]*ImageMetadata, []simplestreams.CloudSpec) {
283
53
284
54
	var toWrite = make([]*ImageMetadata, len(newMetadata))
285
55
	imageIds := make(map[string]bool)
286
56
	for i, im := range newMetadata {
287
57
		newRecord := *im
288
58
		newRecord.Version = seriesVersion
289
59
		newRecord.RegionName = cloudSpec.Region
290
60
		newRecord.Endpoint = cloudSpec.Endpoint
291
61
		toWrite[i] = &newRecord
292
62
		imageIds[mapKey(&newRecord)] = true
293
63
	}
294
64
	regions := make(map[string]bool)
295
65
	var allCloudSpecs = []simplestreams.CloudSpec{*cloudSpec}
296
66
	for _, im := range existingMetadata {
297
67
		if _, ok := imageIds[mapKey(im)]; ok {
298
68
			continue
299
69
		}
300
70
		toWrite = append(toWrite, im)
301
71
		if _, ok := regions[im.RegionName]; ok {
302
72
			continue
303
73
		}
304
74
		regions[im.RegionName] = true
305
75
		existingCloudSpec := simplestreams.CloudSpec{im.RegionName, im.Endpoint}
306
76
		allCloudSpecs = append(allCloudSpecs, existingCloudSpec)
307
77
	}
308
78
	return toWrite, allCloudSpecs
309
79
}
310
80
311
81
type MetadataFile struct {
312
82
	Path string
313
83
	Data []byte
314
84
}
315
85
316
86
// writeMetadata generates some basic simplestreams metadata using the specified cloud and image details and writes
317
87
// it to the supplied store.
318
88
func writeMetadata(metadata []*ImageMetadata, cloudSpec []simplestreams.CloudSpec,
319
89
	metadataStore storage.Storage) error {
320
90
321
91
	index, products, err := MarshalImageMetadataJSON(metadata, cloudSpec, time.Now())
322
92
	if err != nil {
323
93
		return err
324
94
	}
325
95
	metadataInfo := []MetadataFile{
326
96
		{simplestreams.UnsignedIndex, index},
327
97
		{ProductMetadataPath, products},
328
18
	}
98
	}
329
19
	for _, md := range metadataInfo {
99
	for _, md := range metadataInfo {
330
20
		err = metadataStore.Put(md.Path, bytes.NewReader(md.Data), int64(len(md.Data)))
100
		err = metadataStore.Put(md.Path, bytes.NewReader(md.Data), int64(len(md.Data)))
331
@@ -24,29 +104,3 @@
332
24
	}
104
	}
333
25
	return nil
105
	return nil
334
26
}
106
}
335
27
336
28
type MetadataFile struct {
337
29
	Path string
338
30
	Data []byte
339
31
}
340
32
341
33
// generateMetadata generates some basic simplestreams metadata using the specified cloud and image details.
342
34
func generateMetadata(series string, im *ImageMetadata, cloudSpec *simplestreams.CloudSpec) ([]MetadataFile, error) {
343
35
	metadata := &ImageMetadata{
344
36
		Id:         im.Id,
345
37
		Arch:       im.Arch,
346
38
		Release:    series,
347
39
		RegionName: cloudSpec.Region,
348
40
		Endpoint:   cloudSpec.Endpoint,
349
41
	}
350
42
351
43
	index, products, err := MarshalImageMetadataJSON([]*ImageMetadata{metadata}, cloudSpec, time.Now())
352
44
	if err != nil {
353
45
		return nil, err
354
46
	}
355
47
	objects := []MetadataFile{
356
48
		{simplestreams.UnsignedIndex, index},
357
49
		{ProductMetadataPath, products},
358
50
	}
359
51
	return objects, nil
360
52
}
361
53
107
362
=== modified file 'environs/imagemetadata/generate_test.go'
363
--- environs/imagemetadata/generate_test.go	2013-10-16 04:11:38 +0000
364
+++ environs/imagemetadata/generate_test.go	2013-10-17 05:10:43 +0000
365
@@ -4,12 +4,15 @@
366
4
package imagemetadata_test
4
package imagemetadata_test
367
5
5
368
6
import (
6
import (
369
7
	"sort"
370
8
371
7
	gc "launchpad.net/gocheck"
9
	gc "launchpad.net/gocheck"
372
8
10
373
9
	"launchpad.net/juju-core/environs/filestorage"
11
	"launchpad.net/juju-core/environs/filestorage"
374
10
	"launchpad.net/juju-core/environs/imagemetadata"
12
	"launchpad.net/juju-core/environs/imagemetadata"
375
11
	"launchpad.net/juju-core/environs/imagemetadata/testing"
13
	"launchpad.net/juju-core/environs/imagemetadata/testing"
376
12
	"launchpad.net/juju-core/environs/simplestreams"
14
	"launchpad.net/juju-core/environs/simplestreams"
377
15
	"launchpad.net/juju-core/environs/storage"
378
13
	"launchpad.net/juju-core/testing/testbase"
16
	"launchpad.net/juju-core/testing/testbase"
379
14
)
17
)
380
15
18
381
@@ -19,10 +22,27 @@
382
19
	testbase.LoggingSuite
22
	testbase.LoggingSuite
383
20
}
23
}
384
21
24
385
25
func assertFetch(c *gc.C, stor storage.Storage, series, arch, region, endpoint, id string) {
386
26
	cons := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
387
27
		CloudSpec: simplestreams.CloudSpec{region, endpoint},
388
28
		Series:    []string{series},
389
29
		Arches:    []string{arch},
390
30
	})
391
31
	dataSource := storage.NewStorageSimpleStreamsDataSource(stor, "")
392
32
	metadata, err := imagemetadata.Fetch(
393
33
		[]simplestreams.DataSource{dataSource}, simplestreams.DefaultIndexPath, cons, false)
394
34
	c.Assert(err, gc.IsNil)
395
35
	c.Assert(metadata, gc.HasLen, 1)
396
36
	c.Assert(metadata[0].Id, gc.Equals, id)
397
37
}
398
38
399
22
func (s *generateSuite) TestWriteMetadata(c *gc.C) {
39
func (s *generateSuite) TestWriteMetadata(c *gc.C) {
403
23
	im := &imagemetadata.ImageMetadata{
40
	im := []*imagemetadata.ImageMetadata{
404
24
		Id:   "1234",
41
		{
405
25
		Arch: "amd64",
42
			Id:      "1234",
406
43
			Arch:    "amd64",
407
44
			Version: "13.04",
408
45
		},
409
26
	}
46
	}
410
27
	cloudSpec := &simplestreams.CloudSpec{
47
	cloudSpec := &simplestreams.CloudSpec{
411
28
		Region:   "region",
48
		Region:   "region",
412
@@ -31,10 +51,156 @@
413
31
	dir := c.MkDir()
51
	dir := c.MkDir()
414
32
	targetStorage, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
52
	targetStorage, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
415
33
	c.Assert(err, gc.IsNil)
53
	c.Assert(err, gc.IsNil)
417
34
	err = imagemetadata.WriteMetadata("raring", im, cloudSpec, targetStorage)
54
	err = imagemetadata.MergeAndWriteMetadata("raring", im, cloudSpec, targetStorage)
418
35
	c.Assert(err, gc.IsNil)
55
	c.Assert(err, gc.IsNil)
419
36
	metadata := testing.ParseMetadata(c, dir)
56
	metadata := testing.ParseMetadata(c, dir)
420
37
	c.Assert(metadata, gc.HasLen, 1)
57
	c.Assert(metadata, gc.HasLen, 1)
424
38
	im.RegionName = cloudSpec.Region
58
	im[0].RegionName = cloudSpec.Region
425
39
	c.Assert(im, gc.DeepEquals, metadata[0])
59
	im[0].Endpoint = cloudSpec.Endpoint
426
40
}
60
	c.Assert(im[0], gc.DeepEquals, metadata[0])
427
61
	assertFetch(c, targetStorage, "raring", "amd64", "region", "endpoint", "1234")
428
62
}
429
63
430
64
func (s *generateSuite) TestWriteMetadataMergeOverwriteSameArch(c *gc.C) {
431
65
	existingImageMetadata := []*imagemetadata.ImageMetadata{
432
66
		{
433
67
			Id:      "1234",
434
68
			Arch:    "amd64",
435
69
			Version: "13.04",
436
70
		},
437
71
	}
438
72
	cloudSpec := &simplestreams.CloudSpec{
439
73
		Region:   "region",
440
74
		Endpoint: "endpoint",
441
75
	}
442
76
	dir := c.MkDir()
443
77
	targetStorage, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
444
78
	c.Assert(err, gc.IsNil)
445
79
	err = imagemetadata.MergeAndWriteMetadata("raring", existingImageMetadata, cloudSpec, targetStorage)
446
80
	c.Assert(err, gc.IsNil)
447
81
	newImageMetadata := []*imagemetadata.ImageMetadata{
448
82
		{
449
83
			Id:      "abcd",
450
84
			Arch:    "amd64",
451
85
			Version: "13.04",
452
86
		},
453
87
		{
454
88
			Id:      "xyz",
455
89
			Arch:    "arm",
456
90
			Version: "13.04",
457
91
		},
458
92
	}
459
93
	err = imagemetadata.MergeAndWriteMetadata("raring", newImageMetadata, cloudSpec, targetStorage)
460
94
	c.Assert(err, gc.IsNil)
461
95
	metadata := testing.ParseMetadata(c, dir)
462
96
	c.Assert(metadata, gc.HasLen, 2)
463
97
	for _, im := range newImageMetadata {
464
98
		im.RegionName = cloudSpec.Region
465
99
		im.Endpoint = cloudSpec.Endpoint
466
100
	}
467
101
	c.Assert(metadata, gc.DeepEquals, newImageMetadata)
468
102
	assertFetch(c, targetStorage, "raring", "amd64", "region", "endpoint", "abcd")
469
103
	assertFetch(c, targetStorage, "raring", "arm", "region", "endpoint", "xyz")
470
104
}
471
105
472
106
func (s *generateSuite) TestWriteMetadataMergeDifferentSeries(c *gc.C) {
473
107
	existingImageMetadata := []*imagemetadata.ImageMetadata{
474
108
		{
475
109
			Id:      "1234",
476
110
			Arch:    "amd64",
477
111
			Version: "13.04",
478
112
		},
479
113
	}
480
114
	cloudSpec := &simplestreams.CloudSpec{
481
115
		Region:   "region",
482
116
		Endpoint: "endpoint",
483
117
	}
484
118
	dir := c.MkDir()
485
119
	targetStorage, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
486
120
	c.Assert(err, gc.IsNil)
487
121
	err = imagemetadata.MergeAndWriteMetadata("raring", existingImageMetadata, cloudSpec, targetStorage)
488
122
	c.Assert(err, gc.IsNil)
489
123
	newImageMetadata := []*imagemetadata.ImageMetadata{
490
124
		{
491
125
			Id:      "abcd",
492
126
			Arch:    "amd64",
493
127
			Version: "12.04",
494
128
		},
495
129
		{
496
130
			Id:      "xyz",
497
131
			Arch:    "arm",
498
132
			Version: "12.04",
499
133
		},
500
134
	}
501
135
	err = imagemetadata.MergeAndWriteMetadata("precise", newImageMetadata, cloudSpec, targetStorage)
502
136
	c.Assert(err, gc.IsNil)
503
137
	metadata := testing.ParseMetadata(c, dir)
504
138
	c.Assert(metadata, gc.HasLen, 3)
505
139
	newImageMetadata = append(newImageMetadata, existingImageMetadata[0])
506
140
	for _, im := range newImageMetadata {
507
141
		im.RegionName = cloudSpec.Region
508
142
		im.Endpoint = cloudSpec.Endpoint
509
143
	}
510
144
	sort.Sort(byId(metadata))
511
145
	sort.Sort(byId(newImageMetadata))
512
146
	c.Assert(metadata, gc.DeepEquals, newImageMetadata)
513
147
	assertFetch(c, targetStorage, "raring", "amd64", "region", "endpoint", "1234")
514
148
	assertFetch(c, targetStorage, "precise", "amd64", "region", "endpoint", "abcd")
515
149
}
516
150
517
151
func (s *generateSuite) TestWriteMetadataMergeDifferentRegion(c *gc.C) {
518
152
	existingImageMetadata := []*imagemetadata.ImageMetadata{
519
153
		{
520
154
			Id:      "1234",
521
155
			Arch:    "amd64",
522
156
			Version: "13.04",
523
157
		},
524
158
	}
525
159
	cloudSpec := &simplestreams.CloudSpec{
526
160
		Region:   "region",
527
161
		Endpoint: "endpoint",
528
162
	}
529
163
	dir := c.MkDir()
530
164
	targetStorage, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
531
165
	c.Assert(err, gc.IsNil)
532
166
	err = imagemetadata.MergeAndWriteMetadata("raring", existingImageMetadata, cloudSpec, targetStorage)
533
167
	c.Assert(err, gc.IsNil)
534
168
	newImageMetadata := []*imagemetadata.ImageMetadata{
535
169
		{
536
170
			Id:      "abcd",
537
171
			Arch:    "amd64",
538
172
			Version: "13.04",
539
173
		},
540
174
		{
541
175
			Id:      "xyz",
542
176
			Arch:    "arm",
543
177
			Version: "13.04",
544
178
		},
545
179
	}
546
180
	cloudSpec = &simplestreams.CloudSpec{
547
181
		Region:   "region2",
548
182
		Endpoint: "endpoint2",
549
183
	}
550
184
	err = imagemetadata.MergeAndWriteMetadata("raring", newImageMetadata, cloudSpec, targetStorage)
551
185
	c.Assert(err, gc.IsNil)
552
186
	metadata := testing.ParseMetadata(c, dir)
553
187
	c.Assert(metadata, gc.HasLen, 3)
554
188
	for _, im := range newImageMetadata {
555
189
		im.RegionName = "region2"
556
190
		im.Endpoint = "endpoint2"
557
191
	}
558
192
	existingImageMetadata[0].RegionName = "region"
559
193
	existingImageMetadata[0].Endpoint = "endpoint"
560
194
	newImageMetadata = append(newImageMetadata, existingImageMetadata[0])
561
195
	sort.Sort(byId(metadata))
562
196
	sort.Sort(byId(newImageMetadata))
563
197
	c.Assert(metadata, gc.DeepEquals, newImageMetadata)
564
198
	assertFetch(c, targetStorage, "raring", "amd64", "region", "endpoint", "1234")
565
199
	assertFetch(c, targetStorage, "raring", "amd64", "region2", "endpoint2", "abcd")
566
200
}
567
201
568
202
type byId []*imagemetadata.ImageMetadata
569
203
570
204
func (b byId) Len() int           { return len(b) }
571
205
func (b byId) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
572
206
func (b byId) Less(i, j int) bool { return b[i].Id < b[j].Id }
573
41
207
574
=== modified file 'environs/imagemetadata/marshal.go'
575
--- environs/imagemetadata/marshal.go	2013-10-16 04:11:38 +0000
576
+++ environs/imagemetadata/marshal.go	2013-10-17 05:10:43 +0000
577
@@ -18,7 +18,9 @@
578
18
// MarshalImageMetadataJSON marshals image metadata to index and products JSON.
18
// MarshalImageMetadataJSON marshals image metadata to index and products JSON.
579
19
//
19
//
580
20
// updated is the time at which the JSON file was updated.
20
// updated is the time at which the JSON file was updated.
582
21
func MarshalImageMetadataJSON(metadata []*ImageMetadata, cloudSpec *simplestreams.CloudSpec, updated time.Time) (index, products []byte, err error) {
21
func MarshalImageMetadataJSON(metadata []*ImageMetadata, cloudSpec []simplestreams.CloudSpec,
583
22
	updated time.Time) (index, products []byte, err error) {
584
23
585
22
	if index, err = MarshalImageMetadataIndexJSON(metadata, cloudSpec, updated); err != nil {
24
	if index, err = MarshalImageMetadataIndexJSON(metadata, cloudSpec, updated); err != nil {
586
23
		return nil, nil, err
25
		return nil, nil, err
587
24
	}
26
	}
588
@@ -31,13 +33,12 @@
589
31
// MarshalImageMetadataIndexJSON marshals image metadata to index JSON.
33
// MarshalImageMetadataIndexJSON marshals image metadata to index JSON.
590
32
//
34
//
591
33
// updated is the time at which the JSON file was updated.
35
// updated is the time at which the JSON file was updated.
593
34
func MarshalImageMetadataIndexJSON(metadata []*ImageMetadata, cloudSpec *simplestreams.CloudSpec, updated time.Time) (out []byte, err error) {
36
func MarshalImageMetadataIndexJSON(metadata []*ImageMetadata, cloudSpec []simplestreams.CloudSpec,
594
37
	updated time.Time) (out []byte, err error) {
595
38
596
35
	productIds := make([]string, len(metadata))
39
	productIds := make([]string, len(metadata))
597
36
	for i, t := range metadata {
40
	for i, t := range metadata {
602
37
		productIds[i], err = t.productId()
41
		productIds[i] = t.productId()
599
38
		if err != nil {
600
39
			return nil, err
601
40
		}
603
41
	}
42
	}
604
42
	var indices simplestreams.Indices
43
	var indices simplestreams.Indices
605
43
	indices.Updated = updated.Format(time.RFC1123Z)
44
	indices.Updated = updated.Format(time.RFC1123Z)
606
@@ -50,7 +51,7 @@
607
50
			DataType:         "image-ids",
51
			DataType:         "image-ids",
608
51
			ProductsFilePath: ProductMetadataPath,
52
			ProductsFilePath: ProductMetadataPath,
609
52
			ProductIds:       set.NewStrings(productIds...).SortedValues(),
53
			ProductIds:       set.NewStrings(productIds...).SortedValues(),
611
53
			Clouds:           []simplestreams.CloudSpec{*cloudSpec},
54
			Clouds:           cloudSpec,
612
54
		},
55
		},
613
55
	}
56
	}
614
56
	return json.MarshalIndent(&indices, "", "    ")
57
	return json.MarshalIndent(&indices, "", "    ")
615
@@ -66,31 +67,24 @@
616
66
	cloud.Products = make(map[string]simplestreams.MetadataCatalog)
67
	cloud.Products = make(map[string]simplestreams.MetadataCatalog)
617
67
	itemsversion := updated.Format("20060201") // YYYYMMDD
68
	itemsversion := updated.Format("20060201") // YYYYMMDD
618
68
	for _, t := range metadata {
69
	for _, t := range metadata {
619
69
		id, err := t.productId()
620
70
		if err != nil {
621
71
			return nil, err
622
72
		}
623
73
		version, err := simplestreams.SeriesVersion(t.Release)
624
74
		if err != nil {
625
75
			return nil, err
626
76
		}
627
77
		toWrite := &ImageMetadata{
70
		toWrite := &ImageMetadata{
629
78
			Id: t.Id,
71
			Id:         t.Id,
630
72
			RegionName: t.RegionName,
631
73
			Endpoint:   t.Endpoint,
632
79
		}
74
		}
635
80
		if catalog, ok := cloud.Products[id]; ok {
75
		if catalog, ok := cloud.Products[t.productId()]; ok {
636
81
			catalog.Items[itemsversion].Items[t.Id] = t
76
			catalog.Items[itemsversion].Items[t.Id] = toWrite
637
82
		} else {
77
		} else {
638
83
			catalog = simplestreams.MetadataCatalog{
78
			catalog = simplestreams.MetadataCatalog{
642
84
				Arch:       t.Arch,
79
				Arch:    t.Arch,
643
85
				RegionName: t.RegionName,
80
				Version: t.Version,
641
86
				Version:    version,
644
87
				Items: map[string]*simplestreams.ItemCollection{
81
				Items: map[string]*simplestreams.ItemCollection{
645
88
					itemsversion: &simplestreams.ItemCollection{
82
					itemsversion: &simplestreams.ItemCollection{
646
89
						Items: map[string]interface{}{t.Id: toWrite},
83
						Items: map[string]interface{}{t.Id: toWrite},
647
90
					},
84
					},
648
91
				},
85
				},
649
92
			}
86
			}
651
93
			cloud.Products[id] = catalog
87
			cloud.Products[t.productId()] = catalog
652
94
		}
88
		}
653
95
	}
89
	}
654
96
	return json.MarshalIndent(&cloud, "", "    ")
90
	return json.MarshalIndent(&cloud, "", "    ")
655
97
91
656
=== modified file 'environs/imagemetadata/marshal_test.go'
657
--- environs/imagemetadata/marshal_test.go	2013-10-11 06:44:35 +0000
658
+++ environs/imagemetadata/marshal_test.go	2013-10-17 05:10:43 +0000
659
@@ -10,11 +10,14 @@
660
10
10
661
11
	"launchpad.net/juju-core/environs/imagemetadata"
11
	"launchpad.net/juju-core/environs/imagemetadata"
662
12
	"launchpad.net/juju-core/environs/simplestreams"
12
	"launchpad.net/juju-core/environs/simplestreams"
663
13
	"launchpad.net/juju-core/testing/testbase"
664
13
)
14
)
665
14
15
666
15
var _ = gc.Suite(&marshalSuite{})
16
var _ = gc.Suite(&marshalSuite{})
667
16
17
669
17
type marshalSuite struct{}
18
type marshalSuite struct {
670
19
	testbase.LoggingSuite
671
20
}
672
18
21
673
19
var expectedIndex = `{
22
var expectedIndex = `{
674
20
    "index": {
23
    "index": {
675
@@ -90,23 +93,23 @@
676
90
var imageMetadataForTesting = []*imagemetadata.ImageMetadata{
93
var imageMetadataForTesting = []*imagemetadata.ImageMetadata{
677
91
	&imagemetadata.ImageMetadata{
94
	&imagemetadata.ImageMetadata{
678
92
		Id:      "1234",
95
		Id:      "1234",
680
93
		Release: "saucy",
96
		Version: "13.10",
681
94
		Arch:    "arm",
97
		Arch:    "arm",
682
95
	},
98
	},
683
96
	&imagemetadata.ImageMetadata{
99
	&imagemetadata.ImageMetadata{
684
97
		Id:      "5678",
100
		Id:      "5678",
686
98
		Release: "precise",
101
		Version: "12.04",
687
99
		Arch:    "arm",
102
		Arch:    "arm",
688
100
	},
103
	},
689
101
	&imagemetadata.ImageMetadata{
104
	&imagemetadata.ImageMetadata{
690
102
		Id:      "abcd",
105
		Id:      "abcd",
692
103
		Release: "precise",
106
		Version: "12.04",
693
104
		Arch:    "amd64",
107
		Arch:    "amd64",
694
105
	},
108
	},
695
106
}
109
}
696
107
110
697
108
func (s *marshalSuite) TestMarshalIndex(c *gc.C) {
111
func (s *marshalSuite) TestMarshalIndex(c *gc.C) {
699
109
	cloudSpec := &simplestreams.CloudSpec{Region: "region", Endpoint: "endpoint"}
112
	cloudSpec := []simplestreams.CloudSpec{{Region: "region", Endpoint: "endpoint"}}
700
110
	index, err := imagemetadata.MarshalImageMetadataIndexJSON(imageMetadataForTesting, cloudSpec, time.Unix(0, 0).UTC())
113
	index, err := imagemetadata.MarshalImageMetadataIndexJSON(imageMetadataForTesting, cloudSpec, time.Unix(0, 0).UTC())
701
111
	c.Assert(err, gc.IsNil)
114
	c.Assert(err, gc.IsNil)
702
112
	c.Assert(string(index), gc.Equals, expectedIndex)
115
	c.Assert(string(index), gc.Equals, expectedIndex)
703
@@ -119,7 +122,7 @@
704
119
}
122
}
705
120
123
706
121
func (s *marshalSuite) TestMarshal(c *gc.C) {
124
func (s *marshalSuite) TestMarshal(c *gc.C) {
708
122
	cloudSpec := &simplestreams.CloudSpec{Region: "region", Endpoint: "endpoint"}
125
	cloudSpec := []simplestreams.CloudSpec{{Region: "region", Endpoint: "endpoint"}}
709
123
	index, products, err := imagemetadata.MarshalImageMetadataJSON(imageMetadataForTesting, cloudSpec, time.Unix(0, 0).UTC())
126
	index, products, err := imagemetadata.MarshalImageMetadataJSON(imageMetadataForTesting, cloudSpec, time.Unix(0, 0).UTC())
710
124
	c.Assert(err, gc.IsNil)
127
	c.Assert(err, gc.IsNil)
711
125
	c.Assert(string(index), gc.Equals, expectedIndex)
128
	c.Assert(string(index), gc.Equals, expectedIndex)
712
126
129
713
=== modified file 'environs/imagemetadata/simplestreams.go'
714
--- environs/imagemetadata/simplestreams.go	2013-10-17 04:53:59 +0000
715
+++ environs/imagemetadata/simplestreams.go	2013-10-17 05:10:43 +0000
716
@@ -88,9 +88,11 @@
717
88
}
88
}
718
89
89
719
90
func NewImageConstraint(params simplestreams.LookupParams) *ImageConstraint {
90
func NewImageConstraint(params simplestreams.LookupParams) *ImageConstraint {
723
91
	if len(params.Series) != 1 {
91
	if len(params.Series) == 0 {
724
92
		// This can only happen as a result of a coding error.
92
		params.Series = simplestreams.SupportedSeries()
725
93
		panic(fmt.Sprintf("image constraint requires a single series, got %v", params.Series))
93
	}
726
94
	if len(params.Arches) == 0 {
727
95
		params.Arches = []string{"amd64", "i386", "arm"}
728
94
	}
96
	}
729
95
	return &ImageConstraint{LookupParams: params}
97
	return &ImageConstraint{LookupParams: params}
730
96
}
98
}
731
@@ -101,13 +103,18 @@
732
101
	if stream != "" {
103
	if stream != "" {
733
102
		stream = "." + stream
104
		stream = "." + stream
734
103
	}
105
	}
740
104
	version, err := simplestreams.SeriesVersion(ic.Series[0])
106
741
105
	if err != nil {
107
	nrArches := len(ic.Arches)
742
106
		return nil, err
108
	nrSeries := len(ic.Series)
743
107
	}
109
	ids := make([]string, nrArches*nrSeries)
739
108
	ids := make([]string, len(ic.Arches))
744
109
	for i, arch := range ic.Arches {
110
	for i, arch := range ic.Arches {
746
110
		ids[i] = fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, version, arch)
111
		for j, series := range ic.Series {
747
112
			version, err := simplestreams.SeriesVersion(series)
748
113
			if err != nil {
749
114
				return nil, err
750
115
			}
751
116
			ids[j*nrArches+i] = fmt.Sprintf("com.ubuntu.cloud%s:server:%s:%s", stream, version, arch)
752
117
		}
753
111
	}
118
	}
754
112
	return ids, nil
119
	return ids, nil
755
113
}
120
}
756
@@ -118,18 +125,18 @@
757
118
	Storage     string `json:"root_store,omitempty"`
125
	Storage     string `json:"root_store,omitempty"`
758
119
	VType       string `json:"virt,omitempty"`
126
	VType       string `json:"virt,omitempty"`
759
120
	Arch        string `json:"arch,omitempty"`
127
	Arch        string `json:"arch,omitempty"`
761
121
	Release     string `json:"-"`
128
	Version     string `json:"version,omitempty"`
762
122
	RegionAlias string `json:"crsn,omitempty"`
129
	RegionAlias string `json:"crsn,omitempty"`
763
123
	RegionName  string `json:"region,omitempty"`
130
	RegionName  string `json:"region,omitempty"`
764
124
	Endpoint    string `json:"endpoint,omitempty"`
131
	Endpoint    string `json:"endpoint,omitempty"`
765
125
}
132
}
766
126
133
773
127
func (t *ImageMetadata) productId() (string, error) {
134
func (im *ImageMetadata) String() string {
774
128
	seriesVersion, err := simplestreams.SeriesVersion(t.Release)
135
	return fmt.Sprintf("%#v", im)
775
129
	if err != nil {
136
}
776
130
		return "", err
137
777
131
	}
138
func (im *ImageMetadata) productId() string {
778
132
	return fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", seriesVersion, t.Arch), nil
139
	return fmt.Sprintf("com.ubuntu.cloud:server:%s:%s", im.Version, im.Arch)
779
133
}
140
}
780
134
141
781
135
// Fetch returns a list of images for the specified cloud matching the constraint.
142
// Fetch returns a list of images for the specified cloud matching the constraint.
782
@@ -157,6 +164,8 @@
783
157
type imageKey struct {
164
type imageKey struct {
784
158
	vtype   string
165
	vtype   string
785
159
	arch    string
166
	arch    string
786
167
	version string
787
168
	region  string
788
160
	storage string
169
	storage string
789
161
}
170
}
790
162
171
791
@@ -168,14 +177,14 @@
792
168
	imagesMap := make(map[imageKey]*ImageMetadata, len(matchingImages))
177
	imagesMap := make(map[imageKey]*ImageMetadata, len(matchingImages))
793
169
	for _, val := range matchingImages {
178
	for _, val := range matchingImages {
794
170
		im := val.(*ImageMetadata)
179
		im := val.(*ImageMetadata)
796
171
		imagesMap[imageKey{im.VType, im.Arch, im.Storage}] = im
180
		imagesMap[imageKey{im.VType, im.Arch, im.Version, im.RegionName, im.Storage}] = im
797
172
	}
181
	}
798
173
	for _, val := range images {
182
	for _, val := range images {
799
174
		im := val.(*ImageMetadata)
183
		im := val.(*ImageMetadata)
801
175
		if cons.Params().Region != im.RegionName {
184
		if cons != nil && cons.Params().Region != "" && cons.Params().Region != im.RegionName {
802
176
			continue
185
			continue
803
177
		}
186
		}
805
178
		if _, ok := imagesMap[imageKey{im.VType, im.Arch, im.Storage}]; !ok {
187
		if _, ok := imagesMap[imageKey{im.VType, im.Arch, im.Version, im.RegionName, im.Storage}]; !ok {
806
179
			matchingImages = append(matchingImages, im)
188
			matchingImages = append(matchingImages, im)
807
180
		}
189
		}
808
181
	}
190
	}
809
182
191
810
=== modified file 'environs/imagemetadata/simplestreams_test.go'
811
--- environs/imagemetadata/simplestreams_test.go	2013-10-10 11:40:54 +0000
812
+++ environs/imagemetadata/simplestreams_test.go	2013-10-17 05:10:43 +0000
813
@@ -107,91 +107,138 @@
814
107
}
107
}
815
108
108
816
109
var fetchTests = []struct {
109
var fetchTests = []struct {
821
110
	region string
110
	region  string
822
111
	series string
111
	version string
823
112
	arches []string
112
	arches  []string
824
113
	images []*imagemetadata.ImageMetadata
113
	images  []*imagemetadata.ImageMetadata
825
114
}{
114
}{
826
115
	{
115
	{
906
116
		region: "us-east-1",
116
		region:  "us-east-1",
907
117
		series: "precise",
117
		version: "12.04",
908
118
		arches: []string{"amd64", "arm"},
118
		arches:  []string{"amd64", "arm"},
909
119
		images: []*imagemetadata.ImageMetadata{
119
		images: []*imagemetadata.ImageMetadata{
910
120
			{
120
			{
911
121
				Id:         "ami-442ea674",
121
				Id:         "ami-442ea674",
912
122
				VType:      "hvm",
122
				VType:      "hvm",
913
123
				Arch:       "amd64",
123
				Arch:       "amd64",
914
124
				RegionName: "us-east-1",
124
				RegionName: "us-east-1",
915
125
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
125
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
916
126
				Storage:    "ebs",
126
				Storage:    "ebs",
917
127
			},
127
			},
918
128
			{
128
			{
919
129
				Id:         "ami-442ea684",
129
				Id:         "ami-442ea684",
920
130
				VType:      "pv",
130
				VType:      "pv",
921
131
				Arch:       "amd64",
131
				Arch:       "amd64",
922
132
				RegionName: "us-east-1",
132
				RegionName: "us-east-1",
923
133
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
133
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
924
134
				Storage:    "instance",
134
				Storage:    "instance",
925
135
			},
135
			},
926
136
			{
136
			{
927
137
				Id:         "ami-442ea699",
137
				Id:         "ami-442ea699",
928
138
				VType:      "pv",
138
				VType:      "pv",
929
139
				Arch:       "arm",
139
				Arch:       "arm",
930
140
				RegionName: "us-east-1",
140
				RegionName: "us-east-1",
931
141
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
141
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
932
142
				Storage:    "ebs",
142
				Storage:    "ebs",
933
143
			},
143
			},
934
144
		},
144
		},
935
145
	},
145
	},
936
146
	{
146
	{
937
147
		region: "us-east-1",
147
		region:  "us-east-1",
938
148
		series: "precise",
148
		version: "12.04",
939
149
		arches: []string{"amd64"},
149
		arches:  []string{"amd64"},
940
150
		images: []*imagemetadata.ImageMetadata{
150
		images: []*imagemetadata.ImageMetadata{
941
151
			{
151
			{
942
152
				Id:         "ami-442ea674",
152
				Id:         "ami-442ea674",
943
153
				VType:      "hvm",
153
				VType:      "hvm",
944
154
				Arch:       "amd64",
154
				Arch:       "amd64",
945
155
				RegionName: "us-east-1",
155
				RegionName: "us-east-1",
946
156
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
156
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
947
157
				Storage:    "ebs",
157
				Storage:    "ebs",
948
158
			},
158
			},
949
159
			{
159
			{
950
160
				Id:         "ami-442ea684",
160
				Id:         "ami-442ea684",
951
161
				VType:      "pv",
161
				VType:      "pv",
952
162
				Arch:       "amd64",
162
				Arch:       "amd64",
953
163
				RegionName: "us-east-1",
163
				RegionName: "us-east-1",
954
164
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
164
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
955
165
				Storage:    "instance",
165
				Storage:    "instance",
956
166
			},
166
			},
957
167
		},
167
		},
958
168
	},
168
	},
959
169
	{
169
	{
960
170
		region: "us-east-1",
170
		region:  "us-east-1",
961
171
		series: "precise",
171
		version: "12.04",
962
172
		arches: []string{"arm"},
172
		arches:  []string{"arm"},
963
173
		images: []*imagemetadata.ImageMetadata{
173
		images: []*imagemetadata.ImageMetadata{
964
174
			{
174
			{
965
175
				Id:         "ami-442ea699",
175
				Id:         "ami-442ea699",
966
176
				VType:      "pv",
176
				VType:      "pv",
967
177
				Arch:       "arm",
177
				Arch:       "arm",
968
178
				RegionName: "us-east-1",
178
				RegionName: "us-east-1",
969
179
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
179
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
970
180
				Storage:    "ebs",
180
				Storage:    "ebs",
971
181
			},
181
			},
972
182
		},
182
		},
973
183
	},
183
	},
974
184
	{
184
	{
975
185
		region: "us-east-1",
185
		region:  "us-east-1",
976
186
		series: "precise",
186
		version: "12.04",
977
187
		arches: []string{"amd64"},
187
		arches:  []string{"amd64"},
978
188
		images: []*imagemetadata.ImageMetadata{
188
		images: []*imagemetadata.ImageMetadata{
979
189
			{
189
			{
980
190
				Id:         "ami-442ea674",
190
				Id:         "ami-442ea674",
981
191
				VType:      "hvm",
191
				VType:      "hvm",
982
192
				Arch:       "amd64",
192
				Arch:       "amd64",
983
193
				RegionName: "us-east-1",
193
				RegionName: "us-east-1",
984
194
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
194
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
985
195
				Storage:    "ebs",
986
196
			},
987
197
			{
988
198
				Id:         "ami-442ea684",
989
199
				VType:      "pv",
990
200
				Arch:       "amd64",
991
201
				RegionName: "us-east-1",
992
202
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
993
203
				Storage:    "instance",
994
204
			},
995
205
		},
996
206
	},
997
207
	{
998
208
		version: "12.04",
999
209
		arches:  []string{"amd64"},
1000
210
		images: []*imagemetadata.ImageMetadata{
1001
211
			{
1002
212
				Id:         "ami-26745463",
1003
213
				VType:      "pv",
1004
214
				Arch:       "amd64",
1005
215
				RegionName: "au-east-2",
1006
216
				Endpoint:   "https://somewhere-else",
1007
217
				Storage:    "ebs",
1008
218
			},
1009
219
			{
1010
220
				Id:         "ami-442ea674",
1011
221
				VType:      "hvm",
1012
222
				Arch:       "amd64",
1013
223
				RegionName: "us-east-1",
1014
224
				Endpoint:   "https://ec2.us-east-1.amazonaws.com",
1015
225
				Storage:    "ebs",
1016
226
			},
1017
227
			{
1018
228
				Id:          "ami-442ea675",
1019
229
				VType:       "hvm",
1020
230
				Arch:        "amd64",
1021
231
				RegionAlias: "uswest3",
1022
232
				RegionName:  "us-west-3",
1023
233
				Endpoint:    "https://ec2.us-west-3.amazonaws.com",
1024
234
				Storage:     "ebs",
1025
235
			},
1026
236
			{
1027
237
				Id:         "ami-26745464",
1028
238
				VType:      "pv",
1029
239
				Arch:       "amd64",
1030
240
				RegionName: "au-east-1",
1031
241
				Endpoint:   "https://somewhere",
1032
195
				Storage:    "ebs",
242
				Storage:    "ebs",
1033
196
			},
243
			},
1034
197
			{
244
			{
1035
@@ -209,8 +256,12 @@
1036
209
func (s *simplestreamsSuite) TestFetch(c *gc.C) {
256
func (s *simplestreamsSuite) TestFetch(c *gc.C) {
1037
210
	for i, t := range fetchTests {
257
	for i, t := range fetchTests {
1038
211
		c.Logf("test %d", i)
258
		c.Logf("test %d", i)
1039
259
		cloudSpec := simplestreams.CloudSpec{t.region, "https://ec2.us-east-1.amazonaws.com"}
1040
260
		if t.region == "" {
1041
261
			cloudSpec = simplestreams.EmptyCloudSpec
1042
262
		}
1043
212
		imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
263
		imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
1045
213
			CloudSpec: simplestreams.CloudSpec{t.region, "https://ec2.us-east-1.amazonaws.com"},
264
			CloudSpec: cloudSpec,
1046
214
			Series:    []string{"precise"},
265
			Series:    []string{"precise"},
1047
215
			Arches:    t.arches,
266
			Arches:    t.arches,
1048
216
		})
267
		})
1049
@@ -218,6 +269,9 @@
1050
218
		if !c.Check(err, gc.IsNil) {
269
		if !c.Check(err, gc.IsNil) {
1051
219
			continue
270
			continue
1052
220
		}
271
		}
1053
272
		for _, testImage := range t.images {
1054
273
			testImage.Version = t.version
1055
274
		}
1056
221
		c.Check(images, gc.DeepEquals, t.images)
275
		c.Check(images, gc.DeepEquals, t.images)
1057
222
	}
276
	}
1058
223
}
277
}
1059
224
278
1060
=== modified file 'environs/imagemetadata/validation_test.go'
1061
--- environs/imagemetadata/validation_test.go	2013-10-11 06:44:35 +0000
1062
+++ environs/imagemetadata/validation_test.go	2013-10-17 05:10:43 +0000
1063
@@ -20,9 +20,11 @@
1064
20
var _ = gc.Suite(&ValidateSuite{})
20
var _ = gc.Suite(&ValidateSuite{})
1065
21
21
1066
22
func (s *ValidateSuite) makeLocalMetadata(c *gc.C, id, region, series, endpoint string) error {
22
func (s *ValidateSuite) makeLocalMetadata(c *gc.C, id, region, series, endpoint string) error {
1070
23
	im := imagemetadata.ImageMetadata{
23
	metadata := []*imagemetadata.ImageMetadata{
1071
24
		Id:   id,
24
		{
1072
25
		Arch: "amd64",
25
			Id:   id,
1073
26
			Arch: "amd64",
1074
27
		},
1075
26
	}
28
	}
1076
27
	cloudSpec := simplestreams.CloudSpec{
29
	cloudSpec := simplestreams.CloudSpec{
1077
28
		Region:   region,
30
		Region:   region,
1078
@@ -30,7 +32,7 @@
1079
30
	}
32
	}
1080
31
	targetStorage, err := filestorage.NewFileStorageWriter(s.metadataDir, filestorage.UseDefaultTmpDir)
33
	targetStorage, err := filestorage.NewFileStorageWriter(s.metadataDir, filestorage.UseDefaultTmpDir)
1081
32
	c.Assert(err, gc.IsNil)
34
	c.Assert(err, gc.IsNil)
1083
33
	err = imagemetadata.WriteMetadata(series, &im, &cloudSpec, targetStorage)
35
	err = imagemetadata.MergeAndWriteMetadata(series, metadata, &cloudSpec, targetStorage)
1084
34
	if err != nil {
36
	if err != nil {
1085
35
		return err
37
		return err
1086
36
	}
38
	}
1087
37
39
1088
=== modified file 'environs/simplestreams/simplestreams.go'
1089
--- environs/simplestreams/simplestreams.go	2013-10-17 04:53:59 +0000
1090
+++ environs/simplestreams/simplestreams.go	2013-10-17 05:10:43 +0000
1091
@@ -36,6 +36,9 @@
1092
36
	Endpoint string `json:"endpoint"`
36
	Endpoint string `json:"endpoint"`
1093
37
}
37
}
1094
38
38
1095
39
// EmptyCloudSpec is used when we want all records regardless of cloud to be loaded.
1096
40
var EmptyCloudSpec = CloudSpec{}
1097
41
1098
39
// HasRegion is implemented by instances which can provide a region to which they belong.
42
// HasRegion is implemented by instances which can provide a region to which they belong.
1099
40
// A region is defined by region name and endpoint.
43
// A region is defined by region name and endpoint.
1100
41
type HasRegion interface {
44
type HasRegion interface {
1101
@@ -86,6 +89,9 @@
1102
86
89
1103
87
// SeriesVersion returns the version number for the specified Ubuntu series.
90
// SeriesVersion returns the version number for the specified Ubuntu series.
1104
88
func SeriesVersion(series string) (string, error) {
91
func SeriesVersion(series string) (string, error) {
1105
92
	if series == "" {
1106
93
		panic("cannot pass empty series to SeriesVersion()")
1107
94
	}
1108
89
	seriesVersionsMutex.Lock()
95
	seriesVersionsMutex.Lock()
1109
90
	defer seriesVersionsMutex.Unlock()
96
	defer seriesVersionsMutex.Unlock()
1110
91
	if vers, ok := seriesVersions[series]; ok {
97
	if vers, ok := seriesVersions[series]; ok {
1111
@@ -556,13 +562,15 @@
1112
556
	if len(candidates) == 0 {
562
	if len(candidates) == 0 {
1113
557
		return "", errors.NotFoundf("index file missing %q data", indexRef.valueParams.DataType)
563
		return "", errors.NotFoundf("index file missing %q data", indexRef.valueParams.DataType)
1114
558
	}
564
	}
1122
559
	// Restrict by cloud spec.
565
	// Restrict by cloud spec, if required.
1123
560
	hasRightCloud := func(metadata *IndexMetadata) bool {
566
	if cons.Params().CloudSpec != EmptyCloudSpec {
1124
561
		return metadata.hasCloud(cons.Params().CloudSpec)
567
		hasRightCloud := func(metadata *IndexMetadata) bool {
1125
562
	}
568
			return metadata.hasCloud(cons.Params().CloudSpec)
1126
563
	candidates = candidates.filter(hasRightCloud)
569
		}
1127
564
	if len(candidates) == 0 {
570
		candidates = candidates.filter(hasRightCloud)
1128
565
		return "", errors.NotFoundf("index file has no data for cloud %v", cons.Params().CloudSpec)
571
		if len(candidates) == 0 {
1129
572
			return "", errors.NotFoundf("index file has no data for cloud %v", cons.Params().CloudSpec)
1130
573
		}
1131
566
	}
574
	}
1132
567
	// Restrict by product IDs.
575
	// Restrict by product IDs.
1133
568
	hasProduct := func(metadata *IndexMetadata) bool {
576
	hasProduct := func(metadata *IndexMetadata) bool {
1134
@@ -880,8 +888,8 @@
1135
880
888
1136
881
// GetCloudMetadataWithFormat loads the entire cloud metadata encoded using the specified format.
889
// GetCloudMetadataWithFormat loads the entire cloud metadata encoded using the specified format.
1137
882
// Exported for testing.
890
// Exported for testing.
1140
883
func (indexRef *IndexReference) GetCloudMetadataWithFormat(ic LookupConstraint, format string, requireSigned bool) (*CloudMetadata, error) {
891
func (indexRef *IndexReference) GetCloudMetadataWithFormat(cons LookupConstraint, format string, requireSigned bool) (*CloudMetadata, error) {
1141
884
	productFilesPath, err := indexRef.GetProductsPath(ic)
892
	productFilesPath, err := indexRef.GetProductsPath(cons)
1142
885
	if err != nil {
893
	if err != nil {
1143
886
		return nil, err
894
		return nil, err
1144
887
	}
895
	}
Reviewer	Review Type	Date Requested	Status
Juju Engineering		2013-10-14	Pending
Review via email: mp+190898@code.launchpad.net