Merge ~aciba/simplestreams:migrate-to-python3-boto3 into simplestreams:master

[Alberto Contreras]

feat(s3objectstore)!: migrate to python3-boto3

Migrate S3ObjectStore from python3-boto to python3-boto3.

BREAKING CHANGE: Some of the methods did not capture all the underlying
boto expeceptions, and exceptions are different between boto and boto3.
Thus, any callsite handling boto exceptions may have to adjust them.

BREAKING CHANGE: S3ObjectStore.bucket did expose a boto class which is
different in boto3. `bucket` has been removed to not couple
simplestreams' API with boto{,3}'s API. S3ObjectStore.bucketname is
still exposed, so callsites can refer to it however they want.

LP: #2052437

[Alberto Contreras (aciba)]

I have tested this with the following python script. Make sure to have your bucket pre-created and your aws config set.

```py
import errno
import hashlib
from functools import partial

from simplestreams.objectstores.s3 import S3ObjectStore

sos = S3ObjectStore("s3://test-bucket-1712739788/myprefix/")

content = "content"
path = "path"
sos.insert_content(path, content)
sos.insert_content(path, content)

assert sos.source(path).read() == b"content"

md5 = hashlib.md5(content.encode("utf-8")).hexdigest()

assert sos.exists_with_checksum(path, {}) is False
assert sos.exists_with_checksum(path, {"md5": "asdf"}) is False
assert sos.exists_with_checksum(path, {"md5": md5})

sos.remove("path")
sos.remove("path")
sos.remove("path")

assert sos.exists_with_checksum(path, {"md5": md5}) is False

try:
    sos.source(path).read()
except IOError as ex:
    assert ex.errno == errno.ENOENT
    assert str(ex) == f"Unable to open {path}"

with open(path, "wb") as f:
    f.write(content.encode("utf-8"))
sos.insert(path, partial(open, mode="rb"))
assert sos.source(path).read() == b"content"
sos.remove("path")
```

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:a3e76ce0786fc2087f60a0f2541f75c472cc6e0f

No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want jenkins to rebuild you need to trigger it yourself):
https://code.launchpad.net/~aciba/simplestreams/+git/simplestreams/+merge/464027/+edit-commit-message

https://jenkins.canonical.com/server-team/job/simplestreams-ci/35/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-amd64/35/
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-arm64/35/
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-ppc64el/35/
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-s390x/35/

Click here to trigger a rebuild:
https://jenkins.canonical.com/server-team/job/simplestreams-ci/35//rebuild

review: Needs Fixing (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:a3e76ce0786fc2087f60a0f2541f75c472cc6e0f
https://jenkins.canonical.com/server-team/job/simplestreams-ci/36/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-amd64/36/
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-arm64/36/
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-ppc64el/36/
    SUCCESS: https://jenkins.canonical.com/server-team/job/simplestreams-ci/nodes=metal-s390x/36/

Click here to trigger a rebuild:
https://jenkins.canonical.com/server-team/job/simplestreams-ci/36//rebuild

review: Approve (continuous-integration)

[Paride Legovini (paride)]

LGTM, thanks!

review: Approve

diff --git a/debian/control b/debian/control
index 51372b8..a595408 100644
--- a/debian/control
+++ b/debian/control
@@ -31,7 +31,7 @@ Description: Library and tools for using Simple Streams data
 
 Package: python3-simplestreams
 Architecture: all
-Depends: gnupg, python3-boto, ${misc:Depends}, ${python3:Depends}
+Depends: gnupg, python3-boto3, ${misc:Depends}, ${python3:Depends}
 Suggests: python3-requests (>= 1.1)
 Description: Library and tools for using Simple Streams data
  This package provides a client for interacting with simple
diff --git a/simplestreams/objectstores/s3.py b/simplestreams/objectstores/s3.py
index f1e9602..da771f9 100644
--- a/simplestreams/objectstores/s3.py
+++ b/simplestreams/objectstores/s3.py
@@ -15,22 +15,17 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with Simplestreams.  If not, see <http://www.gnu.org/licenses/>.
 
-import boto.exception
-import boto.s3
-import boto.s3.connection
-from contextlib import closing
 import errno
 import tempfile
 
+import boto3
+
 import simplestreams.objectstores as objectstores
 import simplestreams.contentsource as cs
 
 
 class S3ObjectStore(objectstores.ObjectStore):
 
-    _bucket = None
-    _connection = None
-
     def __init__(self, prefix):
         # expect 's3://bucket/path_prefix'
         self.prefix = prefix
@@ -40,19 +35,7 @@ class S3ObjectStore(objectstores.ObjectStore):
             path = prefix
 
         (self.bucketname, self.path_prefix) = path.split("/", 1)
-
-    @property
-    def _conn(self):
-        if not self._connection:
-            self._connection = boto.s3.connection.S3Connection()
-
-        return self._connection
-
-    @property
-    def bucket(self):
-        if not self._bucket:
-            self._bucket = self._conn.get_bucket(self.bucketname)
-        return self._bucket
+        self._client = boto3.client("s3")
 
     def insert(self, path, reader, checksums=None, mutable=True, size=None):
         # store content from reader.read() into path, expecting result checksum
@@ -64,36 +47,54 @@ class S3ObjectStore(objectstores.ObjectStore):
                     tfile.write(buf)
                     if len(buf) != self.read_size:
                         break
-            with closing(self.bucket.new_key(self.path_prefix + path)) as key:
-                key.set_contents_from_file(tfile)
+            tfile.seek(0)
+            self.insert_content(
+                path, tfile, checksums=checksums, mutable=mutable
+            )
         finally:
             tfile.close()
 
     def insert_content(self, path, content, checksums=None, mutable=True):
-        with closing(self.bucket.new_key(self.path_prefix + path)) as key:
-            key.set_contents_from_string(content)
+        self._client.put_object(
+            Body=content, Bucket=self.bucketname, Key=self.path_prefix + path
+        )
 
     def remove(self, path):
         # remove path from store
-        self.bucket.delete_key(self.path_prefix + path)
+        self._client.delete_object(
+            Bucket=self.bucketname, Key=self.path_prefix + path
+        )
 
     def source(self, path):
         # essentially return an 'open(path, r)'
-        key = self.bucket.get_key(self.path_prefix + path)
-        if not key:
+        try:
+            obj_resp = self._client.get_object(
+                Bucket=self.bucketname, Key=self.path_prefix + path
+            )
+        except self._client.exceptions.NoSuchKey:
+            fd = None
+        else:
+            fd = obj_resp.get("Body")
+        if not fd:
             myerr = IOError("Unable to open %s" % path)
             myerr.errno = errno.ENOENT
             raise myerr
 
-        return cs.FdContentSource(fd=key, url=self.path_prefix + path)
+        return cs.FdContentSource(fd=fd, url=self.path_prefix + path)
 
     def exists_with_checksum(self, path, checksums=None):
-        key = self.bucket.get_key(self.path_prefix + path)
-        if key is None:
+        try:
+            obj_resp = self._client.get_object(
+                Bucket=self.bucketname, Key=self.path_prefix + path
+            )
+        except self._client.exceptions.NoSuchKey:
             return False
 
-        if 'md5' in checksums:
-            return checksums['md5'] == key.etag.replace('"', "")
+        if "md5" in checksums:
+            md5 = obj_resp["ResponseMetadata"]["HTTPHeaders"]["etag"].replace(
+                '"', ""
+            )
+            return checksums["md5"] == md5
 
         return False