Merge ~barryprice/swoffsite-charms/+git/swoffsite-charm:master into ~swoffsite-charmers/swoffsite-charms/+git/swoffsite-charm:master

[Barry Price]

Run 'make sync' to update swoffsite itself for keystone v3 support

[Haw Loeung (hloeung)]

LGTM

review: Approve (+1)

diff --git a/swoffsite/.bzr/README b/swoffsite/.bzr/README
 deleted file mode 100644
index f82dc1c..0000000
--- a/swoffsite/.bzr/README
+++ /dev/null
@@ -1,3 +0,0 @@
-This is a Bazaar control directory.
-Do not change any files in this directory.
-See http://bazaar.canonical.com/ for more information about Bazaar.
diff --git a/swoffsite/.bzr/branch-format b/swoffsite/.bzr/branch-format
 deleted file mode 100644
index 9eb09b7..0000000
--- a/swoffsite/.bzr/branch-format
+++ /dev/null
@@ -1 +0,0 @@
-Bazaar-NG meta directory, format 1
diff --git a/swoffsite/.bzr/branch/branch.conf b/swoffsite/.bzr/branch/branch.conf
 deleted file mode 100644
index aace28c..0000000
--- a/swoffsite/.bzr/branch/branch.conf
+++ /dev/null
@@ -1 +0,0 @@
-parent_location = bzr+ssh://bazaar.launchpad.net/+branch/swoffsite/
diff --git a/swoffsite/.bzr/branch/format b/swoffsite/.bzr/branch/format
 deleted file mode 100644
index dc392f4..0000000
--- a/swoffsite/.bzr/branch/format
+++ /dev/null
@@ -1 +0,0 @@
-Bazaar Branch Format 7 (needs bzr 1.6)
diff --git a/swoffsite/.bzr/branch/last-revision b/swoffsite/.bzr/branch/last-revision
 deleted file mode 100644
index ee0a4d1..0000000
--- a/swoffsite/.bzr/branch/last-revision
+++ /dev/null
@@ -1 +0,0 @@
-43 [e-mail]-20190523063027-99vwgfct7fj0eo1z
diff --git a/swoffsite/.bzr/branch/tags b/swoffsite/.bzr/branch/tags
 deleted file mode 100644
index e69de29..0000000
--- a/swoffsite/.bzr/branch/tags
+++ /dev/null
diff --git a/swoffsite/.bzr/checkout/conflicts b/swoffsite/.bzr/checkout/conflicts
 deleted file mode 100644
index 0dc2d3a..0000000
--- a/swoffsite/.bzr/checkout/conflicts
+++ /dev/null
@@ -1 +0,0 @@
-BZR conflict list format 1
diff --git a/swoffsite/.bzr/checkout/dirstate b/swoffsite/.bzr/checkout/dirstate
 deleted file mode 100644
index 51ceaf5..0000000
 Binary files a/swoffsite/.bzr/checkout/dirstate and /dev/null differ
diff --git a/swoffsite/.bzr/checkout/format b/swoffsite/.bzr/checkout/format
 deleted file mode 100644
index e0261c7..0000000
--- a/swoffsite/.bzr/checkout/format
+++ /dev/null
@@ -1 +0,0 @@
-Bazaar Working Tree Format 6 (bzr 1.14)
diff --git a/swoffsite/.bzr/checkout/views b/swoffsite/.bzr/checkout/views
 deleted file mode 100644
index e69de29..0000000
--- a/swoffsite/.bzr/checkout/views
+++ /dev/null
diff --git a/swoffsite/.bzr/repository/format b/swoffsite/.bzr/repository/format
 deleted file mode 100644
index b200528..0000000
--- a/swoffsite/.bzr/repository/format
+++ /dev/null
@@ -1 +0,0 @@
-Bazaar repository format 2a (needs bzr 1.16 or later)
diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.cix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.cix
 deleted file mode 100644
index bb19a90..0000000
 Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.cix and /dev/null differ
diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.iix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.iix
 deleted file mode 100644
index 4cfef68..0000000
 Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.iix and /dev/null differ
diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.rix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.rix
 deleted file mode 100644
index 3fb716f..0000000
 Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.rix and /dev/null differ
diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.six b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.six
 deleted file mode 100644
index cf4403b..0000000
 Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.six and /dev/null differ
diff --git a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.tix b/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.tix
 deleted file mode 100644
index 9e875bb..0000000
 Binary files a/swoffsite/.bzr/repository/indices/80cdf3db9d80e5bed1a92485f6128aa2.tix and /dev/null differ
diff --git a/swoffsite/.bzr/repository/pack-names b/swoffsite/.bzr/repository/pack-names
 deleted file mode 100644
index addd39f..0000000
--- a/swoffsite/.bzr/repository/pack-names
+++ /dev/null
@@ -1,7 +0,0 @@
-B+Tree Graph Index 2
-node_ref_lists=0
-key_elements=1
-len=1
-row_lengths=1
-x��1� PgO��
-C���������=�W�j��}����t�D8���`��R }D1�	p%t���`
 \ No newline at end of file
diff --git a/swoffsite/.bzr/repository/packs/80cdf3db9d80e5bed1a92485f6128aa2.pack b/swoffsite/.bzr/repository/packs/80cdf3db9d80e5bed1a92485f6128aa2.pack
 deleted file mode 100644
index 92ef5ea..0000000
 Binary files a/swoffsite/.bzr/repository/packs/80cdf3db9d80e5bed1a92485f6128aa2.pack and /dev/null differ
diff --git a/swoffsite/swoffsite/cli.py b/swoffsite/swoffsite/cli.py
index aa6381f..a468c6b 100644
--- a/swoffsite/swoffsite/cli.py
+++ b/swoffsite/swoffsite/cli.py
@@ -26,10 +26,10 @@ import os
 import sys
 import tempfile
 
-from .async import FakePool, Pool
 from .cleantemp import cleantemp
 from .log import configure_logging, log
 from .mirror import s3_walk, swift_walk, mirror
+from .pool import FakePool, Pool
 
 
 def s3_bucket_name():
@@ -96,6 +96,9 @@ def parse_args():
     parser.add_argument(
         '--reduced-redundancy', dest='reduced_redundancy', action='store_true',
         help='Upload objects with REDUCED_REDUNDANCY class set.')
+    parser.add_argument(
+        '--no-delete', dest='delete', default=True, action='store_false',
+        help='Do not delete anything; only upload objects.')
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument(
         '--to-s3', dest='direction', action='store_const', const='s3',
@@ -143,7 +146,7 @@ def main():
         mirror(swift_walker, s3_walker, direction=args.direction,
                bucket=args.bucket, gpg_key=args.gpg_key,
                dry_run=args.dry_run, recent=args.recent, pool=pool,
-               reduced_redundancy=args.reduced_redundancy)
+               reduced_redundancy=args.reduced_redundancy, delete=args.delete)
     finally:
         pool.terminate()
 
diff --git a/swoffsite/swoffsite/mirror.py b/swoffsite/swoffsite/mirror.py
index cedd1ea..b256991 100644
--- a/swoffsite/swoffsite/mirror.py
+++ b/swoffsite/swoffsite/mirror.py
@@ -19,7 +19,7 @@
 __all__ = ['s3_walk', 'swift_walk', 'mirror_swift_to_s3']
 
 from cStringIO import StringIO
-from datetime import datetime
+from datetime import datetime, timedelta, tzinfo
 import fnmatch
 import multiprocessing
 import os.path
@@ -45,6 +45,23 @@ SWIFT_BATCHSIZE = 1000
 CHUNK_SIZE = 512*1024
 
 
+# In Python 3 we could just use datetime.timezone.utc, but since this is
+# still Python 2 we have to do it ourselves.
+class UTC(tzinfo):
+
+    def utcoffset(self, dt):
+        return timedelta(0)
+
+    def tzname(self, dt):
+        return 'UTC'
+
+    def dst(self, dt):
+        return timedelta(0)
+
+
+utc = UTC()
+
+
 def get_s3_name(container, name, md5):
     assert '/' not in container, 'Container name must not contain / char'
     return '{}/{}.{}.gpg'.format(container, name, md5)
@@ -150,7 +167,7 @@ def get_swift_con(_cons={}):
             key=os.environ.get('OS_PASSWORD'),
             retries=5,
             snet=False,
-            auth_version='2.0',
+            auth_version=os.environ.get('OS_IDENTITY_API_VERSION', '2.0'),
             os_options=dict(
                 tenant_id=os.environ.get('OS_TENANT_ID'),
                 tenant_name=os.environ.get('OS_TENANT_NAME'),
@@ -424,7 +441,8 @@ def del_swift_task(dry_run, swift_obj):
 
 
 def mirror(swift_walker, s3_walker, direction, bucket,
-           gpg_key, dry_run, recent, pool=None, reduced_redundancy=False):
+           gpg_key, dry_run, recent, pool=None, reduced_redundancy=False,
+           delete=True):
     next_swift_obj = next(swift_walker, None)  # (container, dict)
     next_s3_key = next(s3_walker, None)  # A modified boto.s3.key
 
@@ -466,9 +484,8 @@ def mirror(swift_walker, s3_walker, direction, bucket,
             # Object found in Swift that isn't in S3.
             log.debug('{!r} in Swift but not S3'.format(swift_cmp))
             if direction == 's3':
-                lm = iso8601.parse_date(next_swift_obj[1]['last_modified'],
-                                        None)
-                if not recent or datetime.utcnow() > lm + recent:
+                lm = iso8601.parse_date(next_swift_obj[1]['last_modified'])
+                if not recent or datetime.now(utc) > lm + recent:
                     num_new_files += 1
                     pool.do(to_s3_task, dry_run, lock, bucket, gpg_key,
                             next_swift_obj,
@@ -476,8 +493,9 @@ def mirror(swift_walker, s3_walker, direction, bucket,
                 else:
                     num_ignored_files += 1
             elif direction == 'swift':
-                num_dead_files += 1
-                pool.do(del_swift_task, dry_run, next_swift_obj)
+                if delete:
+                    num_dead_files += 1
+                    pool.do(del_swift_task, dry_run, next_swift_obj)
             else:
                 raise NotImplementedError(direction)
             next_swift_obj = next(swift_walker, None)
@@ -486,11 +504,12 @@ def mirror(swift_walker, s3_walker, direction, bucket,
             # Object found in S3 that isn't in Swift.
             log.debug('{!r} in S3 but not Swift'.format(s3_cmp))
             if direction == 's3':
-                num_dead_files += 1
-                s3_deletions.append(next_s3_key.name)
-                if len(s3_deletions) > 500:
-                    pool.do(del_s3_task, dry_run, bucket, s3_deletions)
-                    s3_deletions = []
+                if delete:
+                    num_dead_files += 1
+                    s3_deletions.append(next_s3_key.name)
+                    if len(s3_deletions) > 500:
+                        pool.do(del_s3_task, dry_run, bucket, s3_deletions)
+                        s3_deletions = []
             elif direction == 'swift':
                 num_new_files += 1
                 pool.do(to_swift_task,
@@ -506,14 +525,15 @@ def mirror(swift_walker, s3_walker, direction, bucket,
             next_swift_obj = next(swift_walker, None)
             next_s3_key = next(s3_walker, None)
 
-    if s3_deletions:
+    if delete and s3_deletions:
         # Do any remaining bulk deletions.
         pool.do(del_s3_task, dry_run, bucket, s3_deletions)
 
     cdir = direction.capitalize()
     log.info('{} files in sync'.format(num_unchanged_files))
     log.info('{} files to send to {}'.format(num_new_files, cdir))
-    log.info('{} files to remove from {}'.format(num_dead_files, cdir))
+    if delete:
+        log.info('{} files to remove from {}'.format(num_dead_files, cdir))
     if recent and direction == 's3':
         log.info('{} files ignored (younger than {})'.format(num_ignored_files,
                                                              recent))
diff --git a/swoffsite/swoffsite/async.py b/swoffsite/swoffsite/pool.py
 similarity index 100%
 rename from swoffsite/swoffsite/async.py
 rename to swoffsite/swoffsite/pool.py