swoffsite Charms

Merge ~barryprice/swoffsite-charms/+git/swoffsite-charm:master into ~swoffsite-charmers/swoffsite-charms/+git/swoffsite-charm:master

  1. Git
  2. lp:~barryprice/swoffsite-charms/+git/swoffsite-charm
  3. master
  4. Merge into master
Proposed by Barry Price
Status: Merged
Merged at revision: 31dce45b74a38d359f2ec6c360569ebf59a09235
Proposed branch: ~barryprice/swoffsite-charms/+git/swoffsite-charm:master
Merge into: ~swoffsite-charmers/swoffsite-charms/+git/swoffsite-charm:master
Diff against target: 265 lines (+40/-24)
3 files modified
dev/null (+0/-7)
swoffsite/swoffsite/cli.py (+5/-2)
swoffsite/swoffsite/mirror.py (+35/-15)
Reviewer Review Type Date Requested Status
Haw Loeung +1 Approve
Review via email: mp+453393@code.launchpad.net

Commit message

Run 'make sync' to update swoffsite itself for keystone v3 support

To post a comment you must log in.
Revision history for this message
Haw Loeung (hloeung) wrote :

LGTM

review: Approve (+1)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/swoffsite/.bzr/README b/swoffsite/.bzr/README
2deleted file mode 100644
3index f82dc1c..0000000
4--- a/swoffsite/.bzr/README
5+++ /dev/null
6@@ -1,3 +0,0 @@
7-This is a Bazaar control directory.
8-Do not change any files in this directory.
9-See http://bazaar.canonical.com/ for more information about Bazaar.
10diff --git a/swoffsite/.bzr/branch-format b/swoffsite/.bzr/branch-format
11deleted file mode 100644
12index 9eb09b7..0000000
13--- a/swoffsite/.bzr/branch-format
14+++ /dev/null
15@@ -1 +0,0 @@
16-Bazaar-NG meta directory, format 1
17diff --git a/swoffsite/.bzr/branch/branch.conf b/swoffsite/.bzr/branch/branch.conf
18deleted file mode 100644
19index aace28c..0000000
20--- a/swoffsite/.bzr/branch/branch.conf
21+++ /dev/null
22@@ -1 +0,0 @@
23-parent_location = bzr+ssh://bazaar.launchpad.net/+branch/swoffsite/
24diff --git a/swoffsite/.bzr/branch/format b/swoffsite/.bzr/branch/format
25deleted file mode 100644
26index dc392f4..0000000
27--- a/swoffsite/.bzr/branch/format
28+++ /dev/null
29@@ -1 +0,0 @@
30-Bazaar Branch Format 7 (needs bzr 1.6)
31diff --git a/swoffsite/.bzr/branch/last-revision b/swoffsite/.bzr/branch/last-revision
32deleted file mode 100644
33index ee0a4d1..0000000
34--- a/swoffsite/.bzr/branch/last-revision
35+++ /dev/null
36@@ -1 +0,0 @@
37-43 mergebot@juju-139df4-prod-is-toolbox-0.canonical.com-20190523063027-99vwgfct7fj0eo1z
38diff --git a/swoffsite/.bzr/branch/tags b/swoffsite/.bzr/branch/tags
39deleted file mode 100644
40index e69de29..0000000
41--- a/swoffsite/.bzr/branch/tags
42+++ /dev/null
43diff --git a/swoffsite/.bzr/checkout/conflicts b/swoffsite/.bzr/checkout/conflicts
44deleted file mode 100644
45index 0dc2d3a..0000000
46--- a/swoffsite/.bzr/checkout/conflicts
47+++ /dev/null
48@@ -1 +0,0 @@
49-BZR conflict list format 1
50diff --git a/swoffsite/.bzr/checkout/dirstate b/swoffsite/.bzr/checkout/dirstate
51deleted file mode 100644
52index 51ceaf5..0000000
53Binary files a/swoffsite/.bzr/checkout/dirstate and /dev/null differ
54diff --git a/swoffsite/.bzr/checkout/format b/swoffsite/.bzr/checkout/format
55deleted file mode 100644
56index e0261c7..0000000
57--- a/swoffsite/.bzr/checkout/format
58+++ /dev/null
59@@ -1 +0,0 @@
60-Bazaar Working Tree Format 6 (bzr 1.14)
61diff --git a/swoffsite/.bzr/checkout/views b/swoffsite/.bzr/checkout/views
62deleted file mode 100644
63index e69de29..0000000
64--- a/swoffsite/.bzr/checkout/views
65+++ /dev/null
66diff --git a/swoffsite/.bzr/repository/format b/swoffsite/.bzr/repository/format
67deleted file mode 100644
68index b200528..0000000
69--- a/swoffsite/.bzr/repository/format
70+++ /dev/null
71@@ -1 +0,0 @@
72-Bazaar repository format 2a (needs bzr 1.16 or later)
73diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.cix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.cix
74deleted file mode 100644
75index bb19a90..0000000
76Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.cix and /dev/null differ
77diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.iix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.iix
78deleted file mode 100644
79index 4cfef68..0000000
80Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.iix and /dev/null differ
81diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.rix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.rix
82deleted file mode 100644
83index 3fb716f..0000000
84Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.rix and /dev/null differ
85diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.six b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.six
86deleted file mode 100644
87index cf4403b..0000000
88Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.six and /dev/null differ
89diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.tix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.tix
90deleted file mode 100644
91index 9e875bb..0000000
92Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.tix and /dev/null differ
93diff --git a/swoffsite/.bzr/repository/pack-names b/swoffsite/.bzr/repository/pack-names
94deleted file mode 100644
95index addd39f..0000000
96--- a/swoffsite/.bzr/repository/pack-names
97+++ /dev/null
98@@ -1,7 +0,0 @@
99-B+Tree Graph Index 2
100-node_ref_lists=0
101-key_elements=1
102-len=1
103-row_lengths=1
104-x��1� PgO��
105-C���������=�W�j��}����t�D8���`��R }D1� p%t���`
106\ No newline at end of file
107diff --git a/swoffsite/.bzr/repository/packs/80cdf3db9d80e5bed1a92485f6128aa2.pack b/swoffsite/.bzr/repository/packs/80cdf3db9d80e5bed1a92485f6128aa2.pack
108deleted file mode 100644
109index 92ef5ea..0000000
110Binary files a/swoffsite/.bzr/repository/packs/80cdf3db9d80e5bed1a92485f6128aa2.pack and /dev/null differ
111diff --git a/swoffsite/swoffsite/cli.py b/swoffsite/swoffsite/cli.py
112index aa6381f..a468c6b 100644
113--- a/swoffsite/swoffsite/cli.py
114+++ b/swoffsite/swoffsite/cli.py
115@@ -26,10 +26,10 @@ import os
116 import sys
117 import tempfile
118
119-from .async import FakePool, Pool
120 from .cleantemp import cleantemp
121 from .log import configure_logging, log
122 from .mirror import s3_walk, swift_walk, mirror
123+from .pool import FakePool, Pool
124
125
126 def s3_bucket_name():
127@@ -96,6 +96,9 @@ def parse_args():
128 parser.add_argument(
129 '--reduced-redundancy', dest='reduced_redundancy', action='store_true',
130 help='Upload objects with REDUCED_REDUNDANCY class set.')
131+ parser.add_argument(
132+ '--no-delete', dest='delete', default=True, action='store_false',
133+ help='Do not delete anything; only upload objects.')
134 group = parser.add_mutually_exclusive_group(required=True)
135 group.add_argument(
136 '--to-s3', dest='direction', action='store_const', const='s3',
137@@ -143,7 +146,7 @@ def main():
138 mirror(swift_walker, s3_walker, direction=args.direction,
139 bucket=args.bucket, gpg_key=args.gpg_key,
140 dry_run=args.dry_run, recent=args.recent, pool=pool,
141- reduced_redundancy=args.reduced_redundancy)
142+ reduced_redundancy=args.reduced_redundancy, delete=args.delete)
143 finally:
144 pool.terminate()
145
146diff --git a/swoffsite/swoffsite/mirror.py b/swoffsite/swoffsite/mirror.py
147index cedd1ea..b256991 100644
148--- a/swoffsite/swoffsite/mirror.py
149+++ b/swoffsite/swoffsite/mirror.py
150@@ -19,7 +19,7 @@
151 __all__ = ['s3_walk', 'swift_walk', 'mirror_swift_to_s3']
152
153 from cStringIO import StringIO
154-from datetime import datetime
155+from datetime import datetime, timedelta, tzinfo
156 import fnmatch
157 import multiprocessing
158 import os.path
159@@ -45,6 +45,23 @@ SWIFT_BATCHSIZE = 1000
160 CHUNK_SIZE = 512*1024
161
162
163+# In Python 3 we could just use datetime.timezone.utc, but since this is
164+# still Python 2 we have to do it ourselves.
165+class UTC(tzinfo):
166+
167+ def utcoffset(self, dt):
168+ return timedelta(0)
169+
170+ def tzname(self, dt):
171+ return 'UTC'
172+
173+ def dst(self, dt):
174+ return timedelta(0)
175+
176+
177+utc = UTC()
178+
179+
180 def get_s3_name(container, name, md5):
181 assert '/' not in container, 'Container name must not contain / char'
182 return '{}/{}.{}.gpg'.format(container, name, md5)
183@@ -150,7 +167,7 @@ def get_swift_con(_cons={}):
184 key=os.environ.get('OS_PASSWORD'),
185 retries=5,
186 snet=False,
187- auth_version='2.0',
188+ auth_version=os.environ.get('OS_IDENTITY_API_VERSION', '2.0'),
189 os_options=dict(
190 tenant_id=os.environ.get('OS_TENANT_ID'),
191 tenant_name=os.environ.get('OS_TENANT_NAME'),
192@@ -424,7 +441,8 @@ def del_swift_task(dry_run, swift_obj):
193
194
195 def mirror(swift_walker, s3_walker, direction, bucket,
196- gpg_key, dry_run, recent, pool=None, reduced_redundancy=False):
197+ gpg_key, dry_run, recent, pool=None, reduced_redundancy=False,
198+ delete=True):
199 next_swift_obj = next(swift_walker, None) # (container, dict)
200 next_s3_key = next(s3_walker, None) # A modified boto.s3.key
201
202@@ -466,9 +484,8 @@ def mirror(swift_walker, s3_walker, direction, bucket,
203 # Object found in Swift that isn't in S3.
204 log.debug('{!r} in Swift but not S3'.format(swift_cmp))
205 if direction == 's3':
206- lm = iso8601.parse_date(next_swift_obj[1]['last_modified'],
207- None)
208- if not recent or datetime.utcnow() > lm + recent:
209+ lm = iso8601.parse_date(next_swift_obj[1]['last_modified'])
210+ if not recent or datetime.now(utc) > lm + recent:
211 num_new_files += 1
212 pool.do(to_s3_task, dry_run, lock, bucket, gpg_key,
213 next_swift_obj,
214@@ -476,8 +493,9 @@ def mirror(swift_walker, s3_walker, direction, bucket,
215 else:
216 num_ignored_files += 1
217 elif direction == 'swift':
218- num_dead_files += 1
219- pool.do(del_swift_task, dry_run, next_swift_obj)
220+ if delete:
221+ num_dead_files += 1
222+ pool.do(del_swift_task, dry_run, next_swift_obj)
223 else:
224 raise NotImplementedError(direction)
225 next_swift_obj = next(swift_walker, None)
226@@ -486,11 +504,12 @@ def mirror(swift_walker, s3_walker, direction, bucket,
227 # Object found in S3 that isn't in Swift.
228 log.debug('{!r} in S3 but not Swift'.format(s3_cmp))
229 if direction == 's3':
230- num_dead_files += 1
231- s3_deletions.append(next_s3_key.name)
232- if len(s3_deletions) > 500:
233- pool.do(del_s3_task, dry_run, bucket, s3_deletions)
234- s3_deletions = []
235+ if delete:
236+ num_dead_files += 1
237+ s3_deletions.append(next_s3_key.name)
238+ if len(s3_deletions) > 500:
239+ pool.do(del_s3_task, dry_run, bucket, s3_deletions)
240+ s3_deletions = []
241 elif direction == 'swift':
242 num_new_files += 1
243 pool.do(to_swift_task,
244@@ -506,14 +525,15 @@ def mirror(swift_walker, s3_walker, direction, bucket,
245 next_swift_obj = next(swift_walker, None)
246 next_s3_key = next(s3_walker, None)
247
248- if s3_deletions:
249+ if delete and s3_deletions:
250 # Do any remaining bulk deletions.
251 pool.do(del_s3_task, dry_run, bucket, s3_deletions)
252
253 cdir = direction.capitalize()
254 log.info('{} files in sync'.format(num_unchanged_files))
255 log.info('{} files to send to {}'.format(num_new_files, cdir))
256- log.info('{} files to remove from {}'.format(num_dead_files, cdir))
257+ if delete:
258+ log.info('{} files to remove from {}'.format(num_dead_files, cdir))
259 if recent and direction == 's3':
260 log.info('{} files ignored (younger than {})'.format(num_ignored_files,
261 recent))
262diff --git a/swoffsite/swoffsite/async.py b/swoffsite/swoffsite/pool.py
263similarity index 100%
264rename from swoffsite/swoffsite/async.py
265rename to swoffsite/swoffsite/pool.py

Subscribers

People subscribed via source and target branches