1
=== modified file 'environs/sshstorage/storage.go'
2
--- environs/sshstorage/storage.go	2013-09-18 22:54:32 +0000
3
+++ environs/sshstorage/storage.go	2013-09-20 04:07:37 +0000
4
@@ -21,18 +21,6 @@
5
21
	"launchpad.net/juju-core/utils"
21
	"launchpad.net/juju-core/utils"
6
22
)
22
)
7
23
23
8
24
const (
9
25
	// tmpdir is the name of the subdirectory
10
26
	// inside the remote storage directory where
11
27
	// temporary files are created.
12
28
	tmpdir = "tmp"
13
29
14
30
	// contentdir is the name of the subdirectory
15
31
	// inside the remote storage directory where
16
32
	// files are stored.
17
33
	contentdir = "content"
18
34
)
19
35
20
36
// SSHStorage implements storage.Storage.
24
// SSHStorage implements storage.Storage.
21
37
//
25
//
22
38
// The storage is created under sudo, and ownership given over to the
26
// The storage is created under sudo, and ownership given over to the
23
@@ -42,6 +30,7 @@
24
42
type SSHStorage struct {
30
type SSHStorage struct {
25
43
	host       string
31
	host       string
26
44
	remotepath string
32
	remotepath string
27
33
	tmpdir     string
28
45
34
29
46
	cmd     *exec.Cmd
35
	cmd     *exec.Cmd
30
47
	stdin   io.WriteCloser
36
	stdin   io.WriteCloser
31
@@ -65,12 +54,24 @@
32
65
	flockExclusive flockmode = "-x"
54
	flockExclusive flockmode = "-x"
33
66
)
55
)
34
67
56
35
57
// UseDefaultTmpDir may be passed into NewSSHStorage
36
58
// for the tmpdir argument, to signify that the default
37
59
// value should be used. See NewSSHStorage for more.
38
60
const UseDefaultTmpDir = ""
39
61
40
68
// NewSSHStorage creates a new SSHStorage, connected to the
62
// NewSSHStorage creates a new SSHStorage, connected to the
41
69
// specified host, managing state under the specified remote path.
63
// specified host, managing state under the specified remote path.
46
70
func NewSSHStorage(host string, remotepath string) (*SSHStorage, error) {
64
//
47
71
	contentdir := path.Join(remotepath, contentdir)
65
// A temporary directory may be specified, in which case it should
48
72
	tmpdir := path.Join(remotepath, tmpdir)
66
// be a directory on the same filesystem as the storage directory
49
73
	script := fmt.Sprintf("install -d -g $SUDO_GID -o $SUDO_UID %s %s", contentdir, tmpdir)
67
// to ensure atomic writes. If left unspecified, tmpdir will be
50
68
// assigned a value of storagedir+".tmp".
51
69
//
52
70
// If tmpdir == UseDefaultTmpDir, it will be created when Put is invoked,
53
71
// and will be removed afterwards. If tmpdir != UseDefaultTmpDir, it must
54
72
// already exist, and will never be removed.
55
73
func NewSSHStorage(host, storagedir, tmpdir string) (*SSHStorage, error) {
56
74
	script := fmt.Sprintf("install -d -g $SUDO_GID -o $SUDO_UID %s", utils.ShQuote(storagedir))
57
74
	cmd := sshCommand(host, true, fmt.Sprintf("sudo bash -c '%s'", script))
75
	cmd := sshCommand(host, true, fmt.Sprintf("sudo bash -c '%s'", script))
58
75
	cmd.Stdin = os.Stdin
76
	cmd.Stdin = os.Stdin
59
76
	if out, err := cmd.CombinedOutput(); err != nil {
77
	if out, err := cmd.CombinedOutput(); err != nil {
60
@@ -93,7 +94,8 @@
61
93
	}
94
	}
62
94
	stor := &SSHStorage{
95
	stor := &SSHStorage{
63
95
		host:       host,
96
		host:       host,
65
96
		remotepath: remotepath,
97
		remotepath: storagedir,
66
98
		tmpdir:     tmpdir,
67
97
		cmd:        cmd,
99
		cmd:        cmd,
68
98
		stdin:      stdin,
100
		stdin:      stdin,
69
99
		stdout:     stdout,
101
		stdout:     stdout,
70
@@ -101,13 +103,8 @@
71
101
	}
103
	}
72
102
	cmd.Start()
104
	cmd.Start()
73
103
105
81
104
	// Verify we have write permissions, and set the temporary directory.
106
	// Verify we have write permissions.
82
105
	_, err = stor.runf(
107
	_, err = stor.runf(flockExclusive, "touch %s", utils.ShQuote(storagedir))
76
106
		flockExclusive,
77
107
		"touch %s && export TMPDIR=%s",
78
108
		utils.ShQuote(remotepath),
79
109
		utils.ShQuote(tmpdir),
80
110
	)
83
111
	if err != nil {
108
	if err != nil {
84
112
		stdin.Close()
109
		stdin.Close()
85
113
		stdout.Close()
110
		stdout.Close()
86
@@ -164,9 +161,8 @@
87
164
161
88
165
// path returns a remote absolute path for a storage object name.
162
// path returns a remote absolute path for a storage object name.
89
166
func (s *SSHStorage) path(name string) (string, error) {
163
func (s *SSHStorage) path(name string) (string, error) {
93
167
	contentdir := path.Join(s.remotepath, contentdir)
164
	remotepath := path.Clean(path.Join(s.remotepath, name))
94
168
	remotepath := path.Clean(path.Join(contentdir, name))
165
	if !strings.HasPrefix(remotepath, s.remotepath) {
92
169
	if !strings.HasPrefix(remotepath, contentdir) {
95
170
		return "", fmt.Errorf("%q escapes storage directory", name)
166
		return "", fmt.Errorf("%q escapes storage directory", name)
96
171
	}
167
	}
97
172
	return remotepath, nil
168
	return remotepath, nil
98
@@ -209,10 +205,9 @@
99
209
		return nil, nil
205
		return nil, nil
100
210
	}
206
	}
101
211
	var names []string
207
	var names []string
102
212
	contentdir := path.Join(s.remotepath, contentdir)
103
213
	for _, name := range strings.Split(out, "\n") {
208
	for _, name := range strings.Split(out, "\n") {
104
214
		if strings.HasPrefix(name[len(dir):], prefix) {
209
		if strings.HasPrefix(name[len(dir):], prefix) {
106
215
			names = append(names, name[len(contentdir)+1:])
210
			names = append(names, name[len(s.remotepath)+1:])
107
216
		}
211
		}
108
217
	}
212
	}
109
218
	sort.Strings(names)
213
	sort.Strings(names)
110
@@ -250,9 +245,30 @@
111
250
	}
245
	}
112
251
	encoded := base64.StdEncoding.EncodeToString(buf)
246
	encoded := base64.StdEncoding.EncodeToString(buf)
113
252
	path = utils.ShQuote(path)
247
	path = utils.ShQuote(path)
114
248
115
249
	tmpdir := s.tmpdir
116
250
	if tmpdir == UseDefaultTmpDir {
117
251
		tmpdir = s.remotepath + ".tmp"
118
252
	}
119
253
	tmpdir = utils.ShQuote(tmpdir)
120
254
121
253
	// Write to a temporary file ($TMPFILE), then mv atomically.
255
	// Write to a temporary file ($TMPFILE), then mv atomically.
122
254
	command := fmt.Sprintf("mkdir -p `dirname %s` && base64 -d > $TMPFILE", path)
256
	command := fmt.Sprintf("mkdir -p `dirname %s` && base64 -d > $TMPFILE", path)
124
255
	command = fmt.Sprintf("TMPFILE=`mktemp` && ((%s && mv $TMPFILE %s) || rm -f $TMPFILE)", command, path)
257
	command = fmt.Sprintf(
125
258
		"export TMPDIR=%s && TMPFILE=`mktemp` && ((%s && mv $TMPFILE %s) || rm -f $TMPFILE)",
126
259
		tmpdir, command, path,
127
260
	)
128
261
129
262
	// If UseDefaultTmpDir is passed, then create the
130
263
	// temporary directory, and remove it afterwards.
131
264
	// Otherwise, the temporary directory is expected
132
265
	// to already exist, and is never removed.
133
266
	if s.tmpdir == UseDefaultTmpDir {
134
267
		installTmpdir := fmt.Sprintf("install -d -g $SUDO_GID -o $SUDO_UID %s", tmpdir)
135
268
		removeTmpdir := fmt.Sprintf("rm -fr %s", tmpdir)
136
269
		command = fmt.Sprintf("%s && (%s); rc=$?; %s; exit $rc", installTmpdir, command, removeTmpdir)
137
270
	}
138
271
139
256
	command = fmt.Sprintf("(%s) << EOF\n%s\nEOF", command, encoded)
272
	command = fmt.Sprintf("(%s) << EOF\n%s\nEOF", command, encoded)
140
257
	_, err = s.runf(flockExclusive, command+"\n")
273
	_, err = s.runf(flockExclusive, command+"\n")
141
258
	return err
274
	return err
142
@@ -271,7 +287,6 @@
143
271
287
144
272
// RemoveAll implements storage.StorageWriter.RemoveAll
288
// RemoveAll implements storage.StorageWriter.RemoveAll
145
273
func (s *SSHStorage) RemoveAll() error {
289
func (s *SSHStorage) RemoveAll() error {
148
274
	contentdir := path.Join(s.remotepath, contentdir)
290
	_, err := s.runf(flockExclusive, "rm -fr %s/*", utils.ShQuote(s.remotepath))
147
275
	_, err := s.runf(flockExclusive, "rm -fr %s/*", utils.ShQuote(contentdir))
149
276
	return err
291
	return err
150
277
}
292
}
151
278
293
152
=== modified file 'environs/sshstorage/storage_test.go'
153
--- environs/sshstorage/storage_test.go	2013-09-20 02:53:59 +0000
154
+++ environs/sshstorage/storage_test.go	2013-09-20 04:07:37 +0000
155
@@ -27,7 +27,6 @@
156
27
27
157
28
type storageSuite struct {
28
type storageSuite struct {
158
29
	testbase.LoggingSuite
29
	testbase.LoggingSuite
159
30
	restoreEnv func()
160
31
}
30
}
161
32
31
162
33
var _ = gc.Suite(&storageSuite{})
32
var _ = gc.Suite(&storageSuite{})
163
@@ -36,8 +35,6 @@
164
36
	gc.TestingT(t)
35
	gc.TestingT(t)
165
37
}
36
}
166
38
37
167
39
var sshCommandOrig = sshCommand
168
40
169
41
func sshCommandTesting(host string, tty bool, command string) *exec.Cmd {
38
func sshCommandTesting(host string, tty bool, command string) *exec.Cmd {
170
42
	cmd := exec.Command("bash", "-c", command)
39
	cmd := exec.Command("bash", "-c", command)
171
43
	uid := fmt.Sprint(os.Getuid())
40
	uid := fmt.Sprint(os.Getuid())
172
@@ -59,50 +56,70 @@
173
59
	c.Assert(err, gc.IsNil)
56
	c.Assert(err, gc.IsNil)
174
60
57
175
61
	bin := c.MkDir()
58
	bin := c.MkDir()
177
62
	s.restoreEnv = testbase.PatchEnvironment("PATH", bin+":"+os.Getenv("PATH"))
59
	restoreEnv := testbase.PatchEnvironment("PATH", bin+":"+os.Getenv("PATH"))
178
60
	s.AddSuiteCleanup(func(*gc.C) { restoreEnv() })
179
63
61
180
64
	// Create a "sudo" command which just executes its args.
62
	// Create a "sudo" command which just executes its args.
183
65
	c.Assert(os.Symlink("/usr/bin/env", filepath.Join(bin, "sudo")), gc.IsNil)
63
	err = os.Symlink("/usr/bin/env", filepath.Join(bin, "sudo"))
184
66
	sshCommand = sshCommandTesting
64
	c.Assert(err, gc.IsNil)
185
65
	restoreSshCommand := testbase.PatchValue(&sshCommand, sshCommandTesting)
186
66
	s.AddSuiteCleanup(func(*gc.C) { restoreSshCommand() })
187
67
67
188
68
	// Create a new "flock" which calls the original, but in non-blocking mode.
68
	// Create a new "flock" which calls the original, but in non-blocking mode.
189
69
	data := []byte(fmt.Sprintf("#!/bin/sh\nexec %s --nonblock \"$@\"", flockBin))
69
	data := []byte(fmt.Sprintf("#!/bin/sh\nexec %s --nonblock \"$@\"", flockBin))
197
70
	c.Assert(ioutil.WriteFile(filepath.Join(bin, "flock"), data, 0755), gc.IsNil)
70
	err = ioutil.WriteFile(filepath.Join(bin, "flock"), data, 0755)
198
71
}
71
	c.Assert(err, gc.IsNil)
199
72
72
}
200
73
func (s *storageSuite) TearDownSuite(c *gc.C) {
73
201
74
	sshCommand = sshCommandOrig
74
func (s *storageSuite) makeStorage(c *gc.C) (storage *SSHStorage, storageDir string) {
202
75
	s.restoreEnv()
75
	storageDir = c.MkDir()
203
76
	s.LoggingSuite.TearDownSuite(c)
76
	storage, err := NewSSHStorage("example.com", storageDir, "")
204
77
	c.Assert(err, gc.IsNil)
205
78
	c.Assert(storage, gc.NotNil)
206
79
	s.AddCleanup(func(*gc.C) { storage.Close() })
207
80
	return storage, storageDir
208
81
}
209
82
210
83
// createFiles creates empty files in the storage directory
211
84
// with the given storage names.
212
85
func createFiles(c *gc.C, storageDir string, names ...string) {
213
86
	for _, name := range names {
214
87
		path := filepath.Join(storageDir, filepath.FromSlash(name))
215
88
		dir := filepath.Dir(path)
216
89
		if err := os.MkdirAll(dir, 0755); err != nil {
217
90
			c.Assert(err, jc.Satisfies, os.IsExist)
218
91
		}
219
92
		err := ioutil.WriteFile(path, nil, 0644)
220
93
		c.Assert(err, gc.IsNil)
221
94
	}
222
77
}
95
}
223
78
96
224
79
func (s *storageSuite) TestNewSSHStorage(c *gc.C) {
97
func (s *storageSuite) TestNewSSHStorage(c *gc.C) {
225
80
	storageDir := c.MkDir()
98
	storageDir := c.MkDir()
226
99
	// Run this block twice to ensure NewSSHStorage can reuse
227
100
	// an existing storage location.
228
81
	for i := 0; i < 2; i++ {
101
	for i := 0; i < 2; i++ {
230
82
		stor, err := NewSSHStorage("example.com", storageDir)
102
		stor, err := NewSSHStorage("example.com", storageDir, UseDefaultTmpDir)
231
83
		c.Assert(err, gc.IsNil)
103
		c.Assert(err, gc.IsNil)
232
84
		c.Assert(stor, gc.NotNil)
104
		c.Assert(stor, gc.NotNil)
233
85
		c.Assert(stor.Close(), gc.IsNil)
105
		c.Assert(stor.Close(), gc.IsNil)
234
86
	}
106
	}
236
87
	c.Assert(os.RemoveAll(storageDir), gc.IsNil)
107
	err := os.RemoveAll(storageDir)
237
108
	c.Assert(err, gc.IsNil)
238
88
109
239
89
	// You must have permissions to create the directory.
110
	// You must have permissions to create the directory.
240
90
	storageDir = c.MkDir()
111
	storageDir = c.MkDir()
243
91
	c.Assert(os.Chmod(storageDir, 0555), gc.IsNil)
112
	err = os.Chmod(storageDir, 0555)
244
92
	_, err := NewSSHStorage("example.com", filepath.Join(storageDir))
113
	c.Assert(err, gc.IsNil)
245
114
	_, err = NewSSHStorage("example.com", filepath.Join(storageDir, "subdir"), UseDefaultTmpDir)
246
93
	c.Assert(err, gc.ErrorMatches, "(.|\n)*cannot change owner and permissions of(.|\n)*")
115
	c.Assert(err, gc.ErrorMatches, "(.|\n)*cannot change owner and permissions of(.|\n)*")
247
94
}
116
}
248
95
117
249
96
func (s *storageSuite) TestPathValidity(c *gc.C) {
118
func (s *storageSuite) TestPathValidity(c *gc.C) {
259
97
	storageDir := c.MkDir()
119
	stor, storageDir := s.makeStorage(c)
260
98
	stor, err := NewSSHStorage("example.com", storageDir)
120
	err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
261
99
	c.Assert(err, gc.IsNil)
121
	c.Assert(err, gc.IsNil)
262
100
	defer stor.Close()
122
	createFiles(c, storageDir, "a/b")
254
101
255
102
	c.Assert(os.Mkdir(filepath.Join(storageDir, contentdir, "a"), 0755), gc.IsNil)
256
103
	f, err := os.Create(filepath.Join(storageDir, contentdir, "a", "b"))
257
104
	c.Assert(err, gc.IsNil)
258
105
	c.Assert(f.Close(), gc.IsNil)
263
106
123
264
107
	for _, prefix := range []string{"..", "a/../.."} {
124
	for _, prefix := range []string{"..", "a/../.."} {
265
108
		c.Logf("prefix: %q", prefix)
125
		c.Logf("prefix: %q", prefix)
266
@@ -122,14 +139,12 @@
267
122
}
139
}
268
123
140
269
124
func (s *storageSuite) TestGet(c *gc.C) {
141
func (s *storageSuite) TestGet(c *gc.C) {
272
125
	storageDir := c.MkDir()
142
	stor, storageDir := s.makeStorage(c)
273
126
	stor, err := NewSSHStorage("example.com", storageDir)
143
	data := []byte("abc\000def")
274
144
	err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
275
127
	c.Assert(err, gc.IsNil)
145
	c.Assert(err, gc.IsNil)
276
128
	defer stor.Close()
277
129
	data := []byte("abc\000def")
278
130
	c.Assert(os.Mkdir(filepath.Join(storageDir, contentdir, "a"), 0755), gc.IsNil)
279
131
	for _, name := range []string{"b", filepath.Join("a", "b")} {
146
	for _, name := range []string{"b", filepath.Join("a", "b")} {
281
132
		err = ioutil.WriteFile(filepath.Join(storageDir, contentdir, name), data, 0644)
147
		err = ioutil.WriteFile(filepath.Join(storageDir, name), data, 0644)
282
133
		c.Assert(err, gc.IsNil)
148
		c.Assert(err, gc.IsNil)
283
134
		r, err := storage.Get(stor, name)
149
		r, err := storage.Get(stor, name)
284
135
		c.Assert(err, gc.IsNil)
150
		c.Assert(err, gc.IsNil)
285
@@ -137,21 +152,17 @@
286
137
		c.Assert(err, gc.IsNil)
152
		c.Assert(err, gc.IsNil)
287
138
		c.Assert(out, gc.DeepEquals, data)
153
		c.Assert(out, gc.DeepEquals, data)
288
139
	}
154
	}
289
140
290
141
	_, err = storage.Get(stor, "notthere")
155
	_, err = storage.Get(stor, "notthere")
291
142
	c.Assert(err, jc.Satisfies, coreerrors.IsNotFoundError)
156
	c.Assert(err, jc.Satisfies, coreerrors.IsNotFoundError)
292
143
}
157
}
293
144
158
294
145
func (s *storageSuite) TestPut(c *gc.C) {
159
func (s *storageSuite) TestPut(c *gc.C) {
299
146
	storageDir := c.MkDir()
160
	stor, storageDir := s.makeStorage(c)
296
147
	stor, err := NewSSHStorage("example.com", storageDir)
297
148
	c.Assert(err, gc.IsNil)
298
149
	defer stor.Close()
300
150
	data := []byte("abc\000def")
161
	data := []byte("abc\000def")
301
151
	for _, name := range []string{"b", filepath.Join("a", "b")} {
162
	for _, name := range []string{"b", filepath.Join("a", "b")} {
303
152
		err = stor.Put(name, bytes.NewBuffer(data), int64(len(data)))
163
		err := stor.Put(name, bytes.NewBuffer(data), int64(len(data)))
304
153
		c.Assert(err, gc.IsNil)
164
		c.Assert(err, gc.IsNil)
306
154
		out, err := ioutil.ReadFile(filepath.Join(storageDir, contentdir, name))
165
		out, err := ioutil.ReadFile(filepath.Join(storageDir, name))
307
155
		c.Assert(err, gc.IsNil)
166
		c.Assert(err, gc.IsNil)
308
156
		c.Assert(out, gc.DeepEquals, data)
167
		c.Assert(out, gc.DeepEquals, data)
309
157
	}
168
	}
310
@@ -165,20 +176,15 @@
311
165
}
176
}
312
166
177
313
167
func (s *storageSuite) TestList(c *gc.C) {
178
func (s *storageSuite) TestList(c *gc.C) {
318
168
	storageDir := c.MkDir()
179
	stor, storageDir := s.makeStorage(c)
315
169
	stor, err := NewSSHStorage("example.com", storageDir)
316
170
	c.Assert(err, gc.IsNil)
317
171
	defer stor.Close()
319
172
	s.assertList(c, stor, "", nil)
180
	s.assertList(c, stor, "", nil)
320
173
181
321
174
	// Directories don't show up in List.
182
	// Directories don't show up in List.
324
175
	contentDir := filepath.Join(storageDir, contentdir)
183
	err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
325
176
	c.Assert(os.Mkdir(filepath.Join(contentDir, "a"), 0755), gc.IsNil)
184
	c.Assert(err, gc.IsNil)
326
177
	s.assertList(c, stor, "", nil)
185
	s.assertList(c, stor, "", nil)
327
178
	s.assertList(c, stor, "a", nil)
186
	s.assertList(c, stor, "a", nil)
331
179
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "a", "b1"), nil, 0), gc.IsNil)
187
	createFiles(c, storageDir, "a/b1", "a/b2", "b")
329
180
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "a", "b2"), nil, 0), gc.IsNil)
330
181
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "b"), nil, 0), gc.IsNil)
332
182
	s.assertList(c, stor, "", []string{"a/b1", "a/b2", "b"})
188
	s.assertList(c, stor, "", []string{"a/b1", "a/b2", "b"})
333
183
	s.assertList(c, stor, "a", []string{"a/b1", "a/b2"})
189
	s.assertList(c, stor, "a", []string{"a/b1", "a/b2"})
334
184
	s.assertList(c, stor, "a/b", []string{"a/b1", "a/b2"})
190
	s.assertList(c, stor, "a/b", []string{"a/b1", "a/b2"})
335
@@ -189,15 +195,10 @@
336
189
}
195
}
337
190
196
338
191
func (s *storageSuite) TestRemove(c *gc.C) {
197
func (s *storageSuite) TestRemove(c *gc.C) {
341
192
	storageDir := c.MkDir()
198
	stor, storageDir := s.makeStorage(c)
342
193
	stor, err := NewSSHStorage("example.com", storageDir)
199
	err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
343
194
	c.Assert(err, gc.IsNil)
200
	c.Assert(err, gc.IsNil)
350
195
	defer stor.Close()
201
	createFiles(c, storageDir, "a/b1", "a/b2")
345
196
346
197
	contentDir := filepath.Join(storageDir, contentdir)
347
198
	c.Assert(os.Mkdir(filepath.Join(contentDir, "a"), 0755), gc.IsNil)
348
199
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "a", "b1"), nil, 0), gc.IsNil)
349
200
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "a", "b2"), nil, 0), gc.IsNil)
351
201
	c.Assert(stor.Remove("a"), gc.ErrorMatches, "rm: cannot remove.*Is a directory")
202
	c.Assert(stor.Remove("a"), gc.ErrorMatches, "rm: cannot remove.*Is a directory")
352
202
	s.assertList(c, stor, "", []string{"a/b1", "a/b2"})
203
	s.assertList(c, stor, "", []string{"a/b1", "a/b2"})
353
203
	c.Assert(stor.Remove("a/b"), gc.IsNil) // doesn't exist; not an error
204
	c.Assert(stor.Remove("a/b"), gc.IsNil) // doesn't exist; not an error
354
@@ -209,39 +210,28 @@
355
209
}
210
}
356
210
211
357
211
func (s *storageSuite) TestRemoveAll(c *gc.C) {
212
func (s *storageSuite) TestRemoveAll(c *gc.C) {
360
212
	storageDir := c.MkDir()
213
	stor, storageDir := s.makeStorage(c)
361
213
	stor, err := NewSSHStorage("example.com", storageDir)
214
	err := os.Mkdir(filepath.Join(storageDir, "a"), 0755)
362
214
	c.Assert(err, gc.IsNil)
215
	c.Assert(err, gc.IsNil)
369
215
	defer stor.Close()
216
	createFiles(c, storageDir, "a/b1", "a/b2")
364
216
365
217
	contentDir := filepath.Join(storageDir, contentdir)
366
218
	c.Assert(os.Mkdir(filepath.Join(contentDir, "a"), 0755), gc.IsNil)
367
219
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "a", "b1"), nil, 0), gc.IsNil)
368
220
	c.Assert(ioutil.WriteFile(filepath.Join(contentDir, "a", "b2"), nil, 0), gc.IsNil)
370
221
	s.assertList(c, stor, "", []string{"a/b1", "a/b2"})
217
	s.assertList(c, stor, "", []string{"a/b1", "a/b2"})
371
222
	c.Assert(stor.RemoveAll(), gc.IsNil)
218
	c.Assert(stor.RemoveAll(), gc.IsNil)
372
223
	s.assertList(c, stor, "", nil)
219
	s.assertList(c, stor, "", nil)
373
224
220
374
225
	// RemoveAll does not remove the base storage directory.
221
	// RemoveAll does not remove the base storage directory.
376
226
	_, err = os.Stat(contentDir)
222
	_, err = os.Stat(storageDir)
377
227
	c.Assert(err, gc.IsNil)
223
	c.Assert(err, gc.IsNil)
378
228
}
224
}
379
229
225
380
230
func (s *storageSuite) TestURL(c *gc.C) {
226
func (s *storageSuite) TestURL(c *gc.C) {
385
231
	storageDir := c.MkDir()
227
	stor, storageDir := s.makeStorage(c)
382
232
	stor, err := NewSSHStorage("example.com", storageDir)
383
233
	c.Assert(err, gc.IsNil)
384
234
	defer stor.Close()
386
235
	url, err := stor.URL("a/b")
228
	url, err := stor.URL("a/b")
387
236
	c.Assert(err, gc.IsNil)
229
	c.Assert(err, gc.IsNil)
389
237
	c.Assert(url, gc.Equals, "sftp://example.com/"+path.Join(storageDir, contentdir, "a/b"))
230
	c.Assert(url, gc.Equals, "sftp://example.com/"+path.Join(storageDir, "a/b"))
390
238
}
231
}
391
239
232
397
240
func (s *storageSuite) TestConsistencyStrategy(c *gc.C) {
233
func (s *storageSuite) TestDefaultConsistencyStrategy(c *gc.C) {
398
241
	storageDir := c.MkDir()
234
	stor, _ := s.makeStorage(c)
394
242
	stor, err := NewSSHStorage("example.com", storageDir)
395
243
	c.Assert(err, gc.IsNil)
396
244
	defer stor.Close()
399
245
	c.Assert(stor.DefaultConsistencyStrategy(), gc.Equals, utils.AttemptStrategy{})
235
	c.Assert(stor.DefaultConsistencyStrategy(), gc.Equals, utils.AttemptStrategy{})
400
246
}
236
}
401
247
237
402
@@ -271,22 +261,19 @@
403
271
	//
261
	//
404
272
	// flock exits with exit code 1 if it can't acquire the
262
	// flock exits with exit code 1 if it can't acquire the
405
273
	// lock immediately in non-blocking mode (which the tests force).
263
	// lock immediately in non-blocking mode (which the tests force).
407
274
	_, err := NewSSHStorage("example.com", storageDir)
264
	_, err := NewSSHStorage("example.com", storageDir, UseDefaultTmpDir)
408
275
	c.Assert(err, gc.ErrorMatches, "exit code 1")
265
	c.Assert(err, gc.ErrorMatches, "exit code 1")
409
276
}
266
}
410
277
267
411
278
func (s *storageSuite) TestWithSharedLocks(c *gc.C) {
268
func (s *storageSuite) TestWithSharedLocks(c *gc.C) {
415
279
	storageDir := c.MkDir()
269
	stor, storageDir := s.makeStorage(c)
413
280
	stor, err := NewSSHStorage("example.com", storageDir)
414
281
	c.Assert(err, gc.IsNil)
416
282
270
417
283
	// Get and List should be able to proceed with a shared lock.
271
	// Get and List should be able to proceed with a shared lock.
418
284
	// All other methods should fail.
272
	// All other methods should fail.
421
285
	data := []byte("abc\000def")
273
	createFiles(c, storageDir, "a")
420
286
	c.Assert(ioutil.WriteFile(filepath.Join(storageDir, contentdir, "a"), data, 0644), gc.IsNil)
422
287
274
423
288
	s.flock(c, flockShared, storageDir)
275
	s.flock(c, flockShared, storageDir)
425
289
	_, err = storage.Get(stor, "a")
276
	_, err := storage.Get(stor, "a")
426
290
	c.Assert(err, gc.IsNil)
277
	c.Assert(err, gc.IsNil)
427
291
	_, err = storage.List(stor, "")
278
	_, err = storage.List(stor, "")
428
292
	c.Assert(err, gc.IsNil)
279
	c.Assert(err, gc.IsNil)
429
@@ -296,13 +283,11 @@
430
296
}
283
}
431
297
284
432
298
func (s *storageSuite) TestWithExclusiveLocks(c *gc.C) {
285
func (s *storageSuite) TestWithExclusiveLocks(c *gc.C) {
436
299
	storageDir := c.MkDir()
286
	stor, storageDir := s.makeStorage(c)
434
300
	stor, err := NewSSHStorage("example.com", storageDir)
435
301
	c.Assert(err, gc.IsNil)
437
302
	// None of the methods (apart from URL) should be able to do anything
287
	// None of the methods (apart from URL) should be able to do anything
438
303
	// while an exclusive lock is held.
288
	// while an exclusive lock is held.
439
304
	s.flock(c, flockExclusive, storageDir)
289
	s.flock(c, flockExclusive, storageDir)
441
305
	_, err = stor.URL("a")
290
	_, err := stor.URL("a")
442
306
	c.Assert(err, gc.IsNil)
291
	c.Assert(err, gc.IsNil)
443
307
	c.Assert(stor.Put("a", bytes.NewBuffer(nil), 0), gc.NotNil)
292
	c.Assert(stor.Put("a", bytes.NewBuffer(nil), 0), gc.NotNil)
444
308
	c.Assert(stor.Remove("a"), gc.NotNil)
293
	c.Assert(stor.Remove("a"), gc.NotNil)
445
@@ -312,3 +297,39 @@
446
312
	_, err = storage.List(stor, "")
297
	_, err = storage.List(stor, "")
447
313
	c.Assert(err, gc.NotNil)
298
	c.Assert(err, gc.NotNil)
448
314
}
299
}
449
300
450
301
func (s *storageSuite) TestPutTmpDir(c *gc.C) {
451
302
	stor, storageDir := s.makeStorage(c)
452
303
453
304
	// Put should create and clean up the temporary directory if
454
305
	// tmpdir==UseDefaultTmpDir.
455
306
	err := stor.Put("test-write", bytes.NewReader(nil), 0)
456
307
	c.Assert(err, gc.IsNil)
457
308
	_, err = os.Stat(storageDir + ".tmp")
458
309
	c.Assert(err, jc.Satisfies, os.IsNotExist)
459
310
460
311
	// To deal with recovering from hard failure, UseDefaultTmpDir
461
312
	// doesn't care if the temporary directory already exists. It
462
313
	// still removes it, though.
463
314
	err = os.Mkdir(storageDir+".tmp", 0755)
464
315
	c.Assert(err, gc.IsNil)
465
316
	err = stor.Put("test-write", bytes.NewReader(nil), 0)
466
317
	c.Assert(err, gc.IsNil)
467
318
	_, err = os.Stat(storageDir + ".tmp")
468
319
	c.Assert(err, jc.Satisfies, os.IsNotExist)
469
320
470
321
	// If we explicitly set the temporary directory, it must already exist.
471
322
	stor.Close()
472
323
	stor, err = NewSSHStorage("example.com", storageDir, storageDir+".tmp")
473
324
	defer stor.Close()
474
325
	c.Assert(err, gc.IsNil)
475
326
	err = stor.Put("test-write", bytes.NewReader(nil), 0)
476
327
	c.Assert(err, gc.ErrorMatches, "mktemp: failed to create file.*No such file or directory")
477
328
	err = os.Mkdir(storageDir+".tmp", 0755)
478
329
	c.Assert(err, gc.IsNil)
479
330
	err = stor.Put("test-write", bytes.NewReader(nil), 0)
480
331
	c.Assert(err, gc.IsNil)
481
332
	// Temporary directory should not have been moved.
482
333
	_, err = os.Stat(storageDir + ".tmp")
483
334
	c.Assert(err, gc.IsNil)
484
335
}
Status:	Merged
Approved by:	Andrew Wilkins on 2013-09-20
Approved revision:	no longer in the source branch.
Merged at revision:	1849
Proposed branch:	lp:~axwalk/juju-core/sshstorage-tmpdir
Merge into:	lp:~go-bot/juju-core/trunk
Diff against target:	484 lines (+154/-118) 2 files modified environs/sshstorage/storage.go (+47/-32) environs/sshstorage/storage_test.go (+107/-86)
To merge this branch:	bzr merge lp:~axwalk/juju-core/sshstorage-tmpdir
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
Tim Penhey (community)		2013-09-16	Needs Information on 2013-09-18
Review via email: mp+185717@code.launchpad.net