Merge lp:~abentley/workspace-runner/scp-retry into lp:workspace-runner

Proposed by Aaron Bentley on 2015-06-25
Status: Merged
Merged at revision: 19
Proposed branch: lp:~abentley/workspace-runner/scp-retry
Merge into: lp:workspace-runner
Diff against target: 128 lines (+38/-21)
2 files modified
workspace_runner/__init__.py (+22/-17)
workspace_runner/tests/__init__.py (+16/-4)
To merge this branch: bzr merge lp:~abentley/workspace-runner/scp-retry
Reviewer Review Type Date Requested Status
Curtis Hovey (community) code 2015-06-25 Approve on 2015-06-25
Review via email: mp+263007@code.launchpad.net

Commit message

Retry scp on retcode 1.

Description of the change

This branch updates the workspace runner to retry scp when it exits with status 1.

To post a comment you must log in.
Curtis Hovey (sinzui) wrote :

Thank you.

review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'workspace_runner/__init__.py'
2--- workspace_runner/__init__.py 2015-06-25 15:06:58 +0000
3+++ workspace_runner/__init__.py 2015-06-25 17:06:42 +0000
4@@ -38,23 +38,28 @@
5 return cls(ssh_connection, tempdir)
6
7
8-def retry_ssh(func):
9- """Decorator that retries a function if it looks like SSH failed.
10+def retry_ssh(retcodes=(255,)):
11+ """Return a decorator that retries a function if it looks like SSH failed.
12
13 SSH connection failures are a CalledProcessError with returncode 255.
14+
15+ Custom retcodes can be specified for operations that have different
16+ failures, like scp, which returns 1.
17 """
18- def decorated(*args, **kwargs):
19- for x in range(3):
20- if x != 0:
21- logging.info('Retrying due to ssh failure.')
22- try:
23- return func(*args, **kwargs)
24- except subprocess.CalledProcessError as e:
25- if e.returncode != 255:
26- raise
27- else:
28- raise
29- return decorated
30+ def decorator(func):
31+ def decorated(*args, **kwargs):
32+ for x in range(3):
33+ if x != 0:
34+ logging.info('Retrying due to ssh failure.')
35+ try:
36+ return func(*args, **kwargs)
37+ except subprocess.CalledProcessError as e:
38+ if e.returncode not in retcodes:
39+ raise
40+ else:
41+ raise
42+ return decorated
43+ return decorator
44
45
46 class SSHConnection:
47@@ -81,7 +86,7 @@
48 command = ['ssh'] + self.get_ssh_options() + [self.host] + args
49 return command
50
51- @retry_ssh
52+ @retry_ssh()
53 def call_with_input(self, args, stdin):
54 """Run a command remotely, accepting stdin."""
55 remote_command = ' '.join(quote(x) for x in args)
56@@ -96,7 +101,7 @@
57 retcode, command, output=output)
58 return output[0]
59
60- @retry_ssh
61+ @retry_ssh((1, 255))
62 def scp_to_remote(self, sources, dest_path):
63 """Copy files to the specified path on the remote host."""
64 dest_loc = '{}:{}/'.format(self.host, quote(dest_path))
65@@ -131,7 +136,7 @@
66 shutil.rmtree(sys.argv[1])
67 """), [self.workspace])
68
69- @retry_ssh
70+ @retry_ssh()
71 def run(self, args, out_file):
72 """Run a command in the workspace."""
73 command = self.ssh_connection.get_ssh_cmd(
74
75=== modified file 'workspace_runner/tests/__init__.py'
76--- workspace_runner/tests/__init__.py 2015-06-25 15:06:58 +0000
77+++ workspace_runner/tests/__init__.py 2015-06-25 17:06:42 +0000
78@@ -130,7 +130,7 @@
79 def test_retry_until_failure(self):
80 attempts = []
81
82- @retry_ssh
83+ @retry_ssh()
84 def fail_repeatedly():
85 attempts.append('attempt')
86 raise subprocess.CalledProcessError(255, [])
87@@ -142,7 +142,7 @@
88 def test_immediate_success(self):
89 attempts = []
90
91- @retry_ssh
92+ @retry_ssh()
93 def succeed():
94 attempts.append('success')
95 return 'success'
96@@ -153,7 +153,7 @@
97 def test_no_retry_non_ssh(self):
98 attempts = []
99
100- @retry_ssh
101+ @retry_ssh()
102 def fail_repeatedly():
103 attempts.append('attempt')
104 raise subprocess.CalledProcessError(254, [])
105@@ -162,10 +162,22 @@
106 fail_repeatedly()
107 self.assertEqual(len(attempts), 1)
108
109+ def test_custom_retcodes(self):
110+ attempts = []
111+
112+ @retry_ssh((254,))
113+ def fail_repeatedly():
114+ attempts.append('attempt')
115+ raise subprocess.CalledProcessError(254, [])
116+
117+ with self.assertRaises(subprocess.CalledProcessError):
118+ fail_repeatedly()
119+ self.assertEqual(len(attempts), 3)
120+
121 def test_nearly_fail(self):
122 attempts = []
123
124- @retry_ssh
125+ @retry_ssh()
126 def nearly_fail():
127 attempts.append('attempt')
128 if len(attempts) == 3:

Subscribers

People subscribed via source and target branches