Merge lp:~fo0bar/turku/turku-agent-cleanup into lp:turku/turku-agent

[Ryan Finnie]

Mega-noop cleanup

This is the minimum required for:
- tox test suite with all passing tests
- black-managed formatting
- Shippable sdist module

It is intended as a base for the other MPs, so they don't have to e.g. establish tests/*, or worry about about existing failing flake8, or worry about how to add additional optional modules.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[lp:~fo0bar/turku/turku-agent-cleanup]

57. By Ryan Finnie on 2020-06-21

Mega-noop cleanup

- Sort imports
- Remove shabangs from non-scripts
- Update MANIFEST.in so `setup.py sdist` produces usable tarballs
- Create stub tests
- Add tox.ini
- Add blank requirements.txt
- Add Makefile
- make black
- Update .bzrignore
- Clean up flake8:
  - update_config.py: '.utils.json_dump_p' imported but unused
  - update_config.py: 'api_reply' is assigned to but never used
  - utils.py: 'sources_secrets_d' is assigned to but never used

[Stuart Bishop (stub)]

Looks good. The new bits seem fine (bzr, buildchain, tox). The code changes all appear to be nothing but Black reformatting (as expected).

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 57

=== modified file '.bzrignore'
--- .bzrignore	2019-04-22 01:16:04 +0000
+++ .bzrignore	2020-06-21 22:22:36 +0000
@@ -1,5 +1,61 @@
-*.pyc
-./build/
-./dist/
-./MANIFEST
-./*.egg-info/
+MANIFEST
+.pybuild/
+.pytest_cache/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
 
=== modified file 'MANIFEST.in'
--- MANIFEST.in	2015-03-29 07:59:33 +0000
+++ MANIFEST.in	2020-06-21 22:22:36 +0000
@@ -1,3 +1,16 @@
+include Makefile
+include README
+include requirements.txt
+include tests/*.py
+include tox.ini
 include turku-agent.cron
+include turku-agent-ping
+include turku-agent-ping.service
+include turku-agent-ping.timer
 include turku-agent-rsyncd.conf
+include turku-agent-rsyncd.init-debian
 include turku-agent-rsyncd.service
+include turku-agent-rsyncd-wrapper
+include turku-update-config
+include turku-update-config.service
+include turku-update-config.timer
 
=== added file 'Makefile'
--- Makefile	1970-01-01 00:00:00 +0000
+++ Makefile	2020-06-21 22:22:36 +0000
@@ -0,0 +1,28 @@
+PYTHON := python3
+
+all: build
+
+build:
+	$(PYTHON) setup.py build
+
+lint:
+	$(PYTHON) -mtox -e flake8
+
+test:
+	$(PYTHON) -mtox
+
+test-quick:
+	$(PYTHON) -mtox -e black,flake8,pytest-quick
+
+black-check:
+	$(PYTHON) -mtox -e black
+
+black:
+	$(PYTHON) -mblack $(CURDIR)
+
+install: build
+	$(PYTHON) setup.py install
+
+clean:
+	$(PYTHON) setup.py clean
+	$(RM) -r build MANIFEST
 
=== added file 'requirements.txt'
=== modified file 'setup.py'
--- setup.py	2019-04-22 01:16:04 +0000
+++ setup.py	2020-06-21 22:22:36 +0000
@@ -18,22 +18,22 @@
 import sys
 from setuptools import setup
 
-assert(sys.version_info > (3, 4))
+assert sys.version_info > (3, 4)
 
 
 setup(
-    name='turku_agent',
-    description='Turku backups - client agent',
-    version='0.2.0',
-    author='Ryan Finnie',
-    author_email='[e-mail]',
-    url='https://launchpad.net/turku',
-    packages=['turku_agent'],
+    name="turku_agent",
+    description="Turku backups - client agent",
+    version="0.2.0",
+    author="Ryan Finnie",
+    author_email="[e-mail]",
+    url="https://launchpad.net/turku",
+    packages=["turku_agent"],
     entry_points={
-        'console_scripts': [
-            'turku-agent-ping = turku_agent.ping:main',
-            'turku-agent-rsyncd-wrapper = turku_agent.rsyncd_wrapper:main',
-            'turku-update-config = turku_agent.update_config:main',
-        ],
+        "console_scripts": [
+            "turku-agent-ping = turku_agent.ping:main",
+            "turku-agent-rsyncd-wrapper = turku_agent.rsyncd_wrapper:main",
+            "turku-update-config = turku_agent.update_config:main",
+        ]
     },
 )
 
=== added directory 'tests'
=== added file 'tests/__init__.py'
=== added file 'tests/test_stub.py'
--- tests/test_stub.py	1970-01-01 00:00:00 +0000
+++ tests/test_stub.py	2020-06-21 22:22:36 +0000
@@ -0,0 +1,8 @@
+import unittest
+import warnings
+
+
+class TestStub(unittest.TestCase):
+    def test_stub(self):
+        # pytest doesn't like a tests/ with no tests
+        warnings.warn("Remove this file once unit tests are added")
 
=== added file 'tox.ini'
--- tox.ini	1970-01-01 00:00:00 +0000
+++ tox.ini	2020-06-21 22:22:36 +0000
@@ -0,0 +1,38 @@
+[tox]
+envlist = black, flake8, pytest
+
+[testenv]
+basepython = python
+
+[testenv:black]
+commands = python -mblack --check .
+deps = black
+
+[testenv:flake8]
+commands = python -mflake8
+deps = flake8
+
+[testenv:pytest]
+commands = python -mpytest --cov=turku_agent --cov-report=term-missing
+deps = pytest
+       pytest-cov
+       -r{toxinidir}/requirements.txt
+
+[testenv:pytest-quick]
+commands = python -mpytest -m "not slow"
+deps = pytest
+       -r{toxinidir}/requirements.txt
+
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+    .tox,
+# TODO: remove C901 once complexity is reduced
+ignore = C901,E203,E231,W503
+max-line-length = 120
+max-complexity = 10
+
+[pytest]
+markers =
+    slow
 
=== modified file 'turku_agent/ping.py'
--- turku_agent/ping.py	2020-03-26 06:06:23 +0000
+++ turku_agent/ping.py	2020-06-21 22:22:36 +0000
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 # Turku backups - client agent
 # Copyright 2015 Canonical Ltd.
 #
@@ -16,13 +14,14 @@
 # this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import json
 import os
-import json
 import random
 import shlex
 import subprocess
 import tempfile
 import time
+
 from .utils import load_config, acquire_lock, api_call
 
 
@@ -30,42 +29,60 @@
     import argparse
 
     parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--config-dir', '-c', type=str, default='/etc/turku-agent')
-    parser.add_argument('--wait', '-w', type=float)
-    parser.add_argument('--restore', action='store_true')
-    parser.add_argument('--restore-storage', type=str, default=None)
-    parser.add_argument('--gonogo-program', type=str, default=None,
-                        help='Go/no-go program run each time to determine whether to ping')
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config-dir", "-c", type=str, default="/etc/turku-agent")
+    parser.add_argument("--wait", "-w", type=float)
+    parser.add_argument("--restore", action="store_true")
+    parser.add_argument("--restore-storage", type=str, default=None)
+    parser.add_argument(
+        "--gonogo-program",
+        type=str,
+        default=None,
+        help="Go/no-go program run each time to determine whether to ping",
+    )
     return parser.parse_args()
 
 
 def call_ssh(config, storage, ssh_req):
     # Write the server host public key
-    t = tempfile.NamedTemporaryFile(mode='w+', encoding='UTF-8')
-    for key in storage['ssh_ping_host_keys']:
-        t.write('%s %s\n' % (storage['ssh_ping_host'], key))
+    t = tempfile.NamedTemporaryFile(mode="w+", encoding="UTF-8")
+    for key in storage["ssh_ping_host_keys"]:
+        t.write("%s %s\n" % (storage["ssh_ping_host"], key))
     t.flush()
 
     # Call ssh
-    ssh_command = config['ssh_command']
+    ssh_command = config["ssh_command"]
     ssh_command += [
-        '-T',
-        '-o', 'BatchMode=yes',
-        '-o', 'UserKnownHostsFile=%s' % t.name,
-        '-o', 'StrictHostKeyChecking=yes',
-        '-o', 'CheckHostIP=no',
-        '-i', config['ssh_private_key_file'],
-        '-R', '%d:%s:%d' % (ssh_req['port'], config['rsyncd_local_address'], config['rsyncd_local_port']),
-        '-p', str(storage['ssh_ping_port']),
-        '-l', storage['ssh_ping_user'],
-        storage['ssh_ping_host'],
-        'turku-ping-remote',
+        "-T",
+        "-o",
+        "BatchMode=yes",
+        "-o",
+        "UserKnownHostsFile=%s" % t.name,
+        "-o",
+        "StrictHostKeyChecking=yes",
+        "-o",
+        "CheckHostIP=no",
+        "-i",
+        config["ssh_private_key_file"],
+        "-R",
+        "%d:%s:%d"
+        % (
+            ssh_req["port"],
+            config["rsyncd_local_address"],
+            config["rsyncd_local_port"],
+        ),
+        "-p",
+        str(storage["ssh_ping_port"]),
+        "-l",
+        storage["ssh_ping_user"],
+        storage["ssh_ping_host"],
+        "turku-ping-remote",
     ]
     p = subprocess.Popen(ssh_command, stdin=subprocess.PIPE)
 
     # Write the ssh request
-    p.stdin.write((json.dumps(ssh_req) + '\n.\n').encode('UTF-8'))
+    p.stdin.write((json.dumps(ssh_req) + "\n.\n").encode("UTF-8"))
     p.stdin.flush()
 
     # Wait for the server to close the SSH connection
@@ -88,16 +105,18 @@
     config = load_config(args.config_dir)
 
     # Basic checks
-    for i in ('ssh_private_key_file', 'machine_uuid', 'machine_secret', 'api_url'):
+    for i in ("ssh_private_key_file", "machine_uuid", "machine_secret", "api_url"):
         if i not in config:
             return
-    if not os.path.isfile(config['ssh_private_key_file']):
+    if not os.path.isfile(config["ssh_private_key_file"]):
         return
 
     # If a go/no-go program is defined, run it and only go if it exits 0.
     # Example: prevent backups during high-load for sensitive systems:
     #   ['check_load', '-c', '1,5,15']
-    gonogo_program = args.gonogo_program if args.gonogo_program else config['gonogo_program']
+    gonogo_program = (
+        args.gonogo_program if args.gonogo_program else config["gonogo_program"]
+    )
     if isinstance(gonogo_program, (list, tuple)):
         # List, program name first, optional arguments after
         gonogo_program_and_args = list(gonogo_program)
@@ -113,106 +132,111 @@
         except (subprocess.CalledProcessError, OSError):
             return
 
-    lock = acquire_lock(os.path.join(config['lock_dir'], 'turku-agent-ping.lock'))
+    lock = acquire_lock(os.path.join(config["lock_dir"], "turku-agent-ping.lock"))
 
     restore_mode = args.restore
 
     # Check with the API server
     api_out = {}
 
-    machine_merge_map = (
-        ('machine_uuid', 'uuid'),
-        ('machine_secret', 'secret'),
-    )
-    api_out['machine'] = {}
+    machine_merge_map = (("machine_uuid", "uuid"), ("machine_secret", "secret"))
+    api_out["machine"] = {}
     for a, b in machine_merge_map:
         if a in config:
-            api_out['machine'][b] = config[a]
+            api_out["machine"][b] = config[a]
 
     if restore_mode:
-        print('Entering restore mode.')
+        print("Entering restore mode.")
         print()
-        api_reply = api_call(config['api_url'], 'agent_ping_restore', api_out)
+        api_reply = api_call(config["api_url"], "agent_ping_restore", api_out)
 
         sources_by_storage = {}
-        for source_name in api_reply['machine']['sources']:
-            source = api_reply['machine']['sources'][source_name]
-            if source_name not in config['sources']:
-                continue
-            if 'storage' not in source:
-                continue
-            if source['storage']['name'] not in sources_by_storage:
-                sources_by_storage[source['storage']['name']] = {}
-            sources_by_storage[source['storage']['name']][source_name] = source
+        for source_name in api_reply["machine"]["sources"]:
+            source = api_reply["machine"]["sources"][source_name]
+            if source_name not in config["sources"]:
+                continue
+            if "storage" not in source:
+                continue
+            if source["storage"]["name"] not in sources_by_storage:
+                sources_by_storage[source["storage"]["name"]] = {}
+            sources_by_storage[source["storage"]["name"]][source_name] = source
 
         if len(sources_by_storage) == 0:
-            print('Cannot find any appropraite sources.')
+            print("Cannot find any appropraite sources.")
             return
-        print('This machine\'s sources are on the following storage units:')
+        print("This machine's sources are on the following storage units:")
         for storage_name in sources_by_storage:
-            print('    %s' % storage_name)
+            print("    %s" % storage_name)
             for source_name in sources_by_storage[storage_name]:
-                print('        %s' % source_name)
+                print("        %s" % source_name)
         print()
         if len(sources_by_storage) == 1:
-            storage = list(list(sources_by_storage.values())[0].values())[0]['storage']
+            storage = list(list(sources_by_storage.values())[0].values())[0]["storage"]
         elif args.restore_storage:
             if args.restore_storage in sources_by_storage:
-                storage = sources_by_storage[args.restore_storage]['storage']
+                storage = sources_by_storage[args.restore_storage]["storage"]
             else:
                 print('Cannot find appropriate storage "%s"' % args.restore_storage)
                 return
         else:
-            print('Multiple storages found.  Please use --restore-storage to specify one.')
+            print(
+                "Multiple storages found.  Please use --restore-storage to specify one."
+            )
             return
 
         ssh_req = {
-            'verbose': True,
-            'action': 'restore',
-            'port': random.randint(49152, 65535),
+            "verbose": True,
+            "action": "restore",
+            "port": random.randint(49152, 65535),
         }
-        print('Storage unit: %s' % storage['name'])
-        if 'restore_path' in config:
-            print('Local destination path: %s' % config['restore_path'])
-            print('Sample restore usage from storage unit:')
+        print("Storage unit: %s" % storage["name"])
+        if "restore_path" in config:
+            print("Local destination path: %s" % config["restore_path"])
+            print("Sample restore usage from storage unit:")
             print(
-                '    RSYNC_PASSWORD=%s rsync -avzP --numeric-ids ${P?}/ rsync://%s@127.0.0.1:%s/%s/' % (
-                    config['restore_password'],
-                    config['restore_username'],
-                    ssh_req['port'], config['restore_module']
+                "    RSYNC_PASSWORD=%s rsync -avzP --numeric-ids ${P?}/ rsync://%s@127.0.0.1:%s/%s/"
+                % (
+                    config["restore_password"],
+                    config["restore_username"],
+                    ssh_req["port"],
+                    config["restore_module"],
                 )
             )
             print()
         call_ssh(config, storage, ssh_req)
     else:
-        api_reply = api_call(config['api_url'], 'agent_ping_checkin', api_out)
+        api_reply = api_call(config["api_url"], "agent_ping_checkin", api_out)
 
-        if 'scheduled_sources' not in api_reply:
+        if "scheduled_sources" not in api_reply:
             return
         sources_by_storage = {}
-        for source_name in api_reply['machine']['scheduled_sources']:
-            source = api_reply['machine']['scheduled_sources'][source_name]
-            if source_name not in config['sources']:
-                continue
-            if 'storage' not in source:
-                continue
-            if source['storage']['name'] not in sources_by_storage:
-                sources_by_storage[source['storage']['name']] = {}
-            sources_by_storage[source['storage']['name']][source_name] = source
+        for source_name in api_reply["machine"]["scheduled_sources"]:
+            source = api_reply["machine"]["scheduled_sources"][source_name]
+            if source_name not in config["sources"]:
+                continue
+            if "storage" not in source:
+                continue
+            if source["storage"]["name"] not in sources_by_storage:
+                sources_by_storage[source["storage"]["name"]] = {}
+            sources_by_storage[source["storage"]["name"]][source_name] = source
 
         for storage_name in sources_by_storage:
             ssh_req = {
-                'verbose': True,
-                'action': 'checkin',
-                'port': random.randint(49152, 65535),
-                'sources': {},
+                "verbose": True,
+                "action": "checkin",
+                "port": random.randint(49152, 65535),
+                "sources": {},
             }
             for source in sources_by_storage[storage_name]:
-                ssh_req['sources'][source] = {
-                    'username': config['sources'][source]['username'],
-                    'password': config['sources'][source]['password'],
+                ssh_req["sources"][source] = {
+                    "username": config["sources"][source]["username"],
+                    "password": config["sources"][source]["password"],
                 }
-            call_ssh(config, list(sources_by_storage[storage_name].values())[0]['storage'], ssh_req)
+            call_ssh(
+                config,
+                list(sources_by_storage[storage_name].values())[0]["storage"],
+                ssh_req,
+            )
 
     # Cleanup
     lock.close()
 
=== modified file 'turku_agent/rsyncd_wrapper.py'
--- turku_agent/rsyncd_wrapper.py	2019-04-22 01:16:04 +0000
+++ turku_agent/rsyncd_wrapper.py	2020-06-21 22:22:36 +0000
@@ -16,6 +16,7 @@
 # this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+
 from .utils import load_config
 
 
@@ -23,9 +24,10 @@
     import argparse
 
     parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--config-dir', '-c', type=str, default='/etc/turku-agent')
-    parser.add_argument('--detach', action='store_true')
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config-dir", "-c", type=str, default="/etc/turku-agent")
+    parser.add_argument("--detach", action="store_true")
     return parser.parse_known_args()
 
 
@@ -33,10 +35,12 @@
     args, rest = parse_args()
 
     config = load_config(args.config_dir)
-    rsyncd_command = config['rsyncd_command']
+    rsyncd_command = config["rsyncd_command"]
     if not args.detach:
-        rsyncd_command.append('--no-detach')
-    rsyncd_command.append('--daemon')
-    rsyncd_command.append('--config=%s' % os.path.join(config['var_dir'], 'rsyncd.conf'))
+        rsyncd_command.append("--no-detach")
+    rsyncd_command.append("--daemon")
+    rsyncd_command.append(
+        "--config=%s" % os.path.join(config["var_dir"], "rsyncd.conf")
+    )
     rsyncd_command += rest
     os.execvp(rsyncd_command[0], rsyncd_command)
 
=== modified file 'turku_agent/update_config.py'
--- turku_agent/update_config.py	2019-05-23 14:18:25 +0000
+++ turku_agent/update_config.py	2020-06-21 22:22:36 +0000
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 # Turku backups - client agent
 # Copyright 2015 Canonical Ltd.
 #
@@ -15,12 +13,13 @@
 # You should have received a copy of the GNU General Public License along with
 # this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import os
 import random
-import os
 import subprocess
 import time
-import logging
-from .utils import json_dump_p, json_dumps_p, load_config, fill_config, acquire_lock, api_call
+
+from .utils import json_dumps_p, load_config, fill_config, acquire_lock, api_call
 
 
 class IncompleteConfigError(Exception):
@@ -31,69 +30,70 @@
     import argparse
 
     parser = argparse.ArgumentParser(
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--config-dir', '-c', type=str, default='/etc/turku-agent')
-    parser.add_argument('--wait', '-w', type=float)
-    parser.add_argument('--debug', action='store_true')
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument("--config-dir", "-c", type=str, default="/etc/turku-agent")
+    parser.add_argument("--wait", "-w", type=float)
+    parser.add_argument("--debug", action="store_true")
     return parser.parse_args()
 
 
 def write_conf_files(config):
     # Build rsyncd.conf
     built_rsyncd_conf = (
-        'address = %s\n' % config['rsyncd_local_address'] +
-        'port = %d\n' % config['rsyncd_local_port'] +
-        'log file = /dev/stdout\n' +
-        'uid = root\n' +
-        'gid = root\n' +
-        'list = false\n\n'
+        "address = %s\n" % config["rsyncd_local_address"]
+        + "port = %d\n" % config["rsyncd_local_port"]
+        + "log file = /dev/stdout\n"
+        + "uid = root\n"
+        + "gid = root\n"
+        + "list = false\n\n"
     )
     rsyncd_secrets = []
-    rsyncd_secrets.append((config['restore_username'], config['restore_password']))
+    rsyncd_secrets.append((config["restore_username"], config["restore_password"]))
     built_rsyncd_conf += (
-        '[%s]\n' +
-        '    path = %s\n' +
-        '    auth users = %s\n' +
-        '    secrets file = %s\n' +
-        '    read only = false\n\n'
+        "[%s]\n"
+        + "    path = %s\n"
+        + "    auth users = %s\n"
+        + "    secrets file = %s\n"
+        + "    read only = false\n\n"
     ) % (
-        config['restore_module'],
-        config['restore_path'],
-        config['restore_username'],
-        os.path.join(config['var_dir'], 'rsyncd.secrets'),
+        config["restore_module"],
+        config["restore_path"],
+        config["restore_username"],
+        os.path.join(config["var_dir"], "rsyncd.secrets"),
     )
-    for s in config['sources']:
-        sd = config['sources'][s]
-        rsyncd_secrets.append((sd['username'], sd['password']))
+    for s in config["sources"]:
+        sd = config["sources"][s]
+        rsyncd_secrets.append((sd["username"], sd["password"]))
         built_rsyncd_conf += (
-            '[%s]\n' +
-            '    path = %s\n' +
-            '    auth users = %s\n' +
-            '    secrets file = %s\n' +
-            '    read only = true\n\n'
+            "[%s]\n"
+            + "    path = %s\n"
+            + "    auth users = %s\n"
+            + "    secrets file = %s\n"
+            + "    read only = true\n\n"
         ) % (
             s,
-            sd['path'],
-            sd['username'],
-            os.path.join(config['var_dir'], 'rsyncd.secrets'),
+            sd["path"],
+            sd["username"],
+            os.path.join(config["var_dir"], "rsyncd.secrets"),
         )
-    with open(os.path.join(config['var_dir'], 'rsyncd.conf'), 'w') as f:
+    with open(os.path.join(config["var_dir"], "rsyncd.conf"), "w") as f:
         f.write(built_rsyncd_conf)
 
     # Build rsyncd.secrets
-    built_rsyncd_secrets = ''
+    built_rsyncd_secrets = ""
     for (username, password) in rsyncd_secrets:
-        built_rsyncd_secrets += username + ':' + password + '\n'
-    with open(os.path.join(config['var_dir'], 'rsyncd.secrets'), 'w') as f:
+        built_rsyncd_secrets += username + ":" + password + "\n"
+    with open(os.path.join(config["var_dir"], "rsyncd.secrets"), "w") as f:
         os.fchmod(f.fileno(), 0o600)
         f.write(built_rsyncd_secrets)
 
 
 def init_is_upstart():
     try:
-        return 'upstart' in subprocess.check_output(
-            ['initctl', 'version'],
-            stderr=subprocess.DEVNULL, universal_newlines=True)
+        return "upstart" in subprocess.check_output(
+            ["initctl", "version"], stderr=subprocess.DEVNULL, universal_newlines=True
+        )
     except (FileNotFoundError, subprocess.CalledProcessError):
         return False
 
@@ -107,52 +107,54 @@
         # With Upstart, start will fail if the service is already running,
         # so we need to check for that first.
         try:
-            if 'start/running' in subprocess.check_output(
-                    ['status', 'turku-agent-rsyncd'],
-                    stderr=subprocess.STDOUT, universal_newlines=True):
+            if "start/running" in subprocess.check_output(
+                ["status", "turku-agent-rsyncd"],
+                stderr=subprocess.STDOUT,
+                universal_newlines=True,
+            ):
                 return
         except subprocess.CalledProcessError:
             pass
-    subprocess.check_call(['service', 'turku-agent-rsyncd', 'start'])
+    subprocess.check_call(["service", "turku-agent-rsyncd", "start"])
 
 
 def send_config(config):
-    required_keys = ['api_url']
-    if 'api_auth' not in config:
-        required_keys += ['api_auth_name', 'api_auth_secret']
+    required_keys = ["api_url"]
+    if "api_auth" not in config:
+        required_keys += ["api_auth_name", "api_auth_secret"]
     for k in required_keys:
         if k not in config:
             raise IncompleteConfigError('Required config "%s" not found.' % k)
 
     api_out = {}
-    if ('api_auth_name' in config) and ('api_auth_secret' in config):
+    if ("api_auth_name" in config) and ("api_auth_secret" in config):
         # name/secret style
-        api_out['auth'] = {
-            'name': config['api_auth_name'],
-            'secret': config['api_auth_secret'],
+        api_out["auth"] = {
+            "name": config["api_auth_name"],
+            "secret": config["api_auth_secret"],
         }
     else:
         # nameless secret style
-        api_out['auth'] = config['api_auth']
+        api_out["auth"] = config["api_auth"]
 
     # Merge the following options into the machine section
     machine_merge_map = (
-        ('machine_uuid', 'uuid'),
-        ('machine_secret', 'secret'),
-        ('environment_name', 'environment_name'),
-        ('service_name', 'service_name'),
-        ('unit_name', 'unit_name'),
-        ('ssh_public_key', 'ssh_public_key'),
-        ('published', 'published'),
+        ("machine_uuid", "uuid"),
+        ("machine_secret", "secret"),
+        ("environment_name", "environment_name"),
+        ("service_name", "service_name"),
+        ("unit_name", "unit_name"),
+        ("ssh_public_key", "ssh_public_key"),
+        ("published", "published"),
     )
-    api_out['machine'] = {}
+    api_out["machine"] = {}
     for a, b in machine_merge_map:
         if a in config:
-            api_out['machine'][b] = config[a]
-
-    api_out['machine']['sources'] = config['sources']
-
-    api_reply = api_call(config['api_url'], 'update_config', api_out)
+            api_out["machine"][b] = config[a]
+
+    api_out["machine"]["sources"] = config["sources"]
+
+    api_call(config["api_url"], "update_config", api_out)
 
 
 def main():
@@ -162,7 +164,7 @@
         time.sleep(random.uniform(0, args.wait))
 
     config = load_config(args.config_dir)
-    lock = acquire_lock(os.path.join(config['lock_dir'], 'turku-update-config.lock'))
+    lock = acquire_lock(os.path.join(config["lock_dir"], "turku-update-config.lock"))
     fill_config(config)
     if args.debug:
         print(json_dumps_p(config))
 
=== modified file 'turku_agent/utils.py'
--- turku_agent/utils.py	2020-03-23 22:31:56 +0000
+++ turku_agent/utils.py	2020-06-21 22:22:36 +0000
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 # Turku backups - client agent
 # Copyright 2015 Canonical Ltd.
 #
@@ -15,32 +13,34 @@
 # You should have received a copy of the GNU General Public License along with
 # this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import uuid
-import string
-import random
+import copy
+import http.client
 import json
 import os
-import copy
+import platform
+import random
+import string
 import subprocess
-import platform
 import urllib.parse
-import http.client
-
-
-class RuntimeLock():
+import uuid
+
+
+class RuntimeLock:
     name = None
     file = None
 
     def __init__(self, name):
         import fcntl
-        file = open(name, 'w')
+
+        file = open(name, "w")
         try:
             fcntl.lockf(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
         except IOError as e:
             import errno
+
             if e.errno in (errno.EACCES, errno.EAGAIN):
                 raise
-        file.write('%10s\n' % os.getpid())
+        file.write("%10s\n" % os.getpid())
         file.flush()
         file.seek(0)
         self.name = name
@@ -71,12 +71,12 @@
 
 def json_dump_p(obj, f):
     """Calls json.dump with standard (pretty) formatting"""
-    return json.dump(obj, f, sort_keys=True, indent=4, separators=(',', ': '))
+    return json.dump(obj, f, sort_keys=True, indent=4, separators=(",", ": "))
 
 
 def json_dumps_p(obj):
     """Calls json.dumps with standard (pretty) formatting"""
-    return json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': '))
+    return json.dumps(obj, sort_keys=True, indent=4, separators=(",", ": "))
 
 
 def json_load_file(file):
@@ -103,10 +103,10 @@
 
 def load_config(config_dir):
     config = {}
-    config['config_dir'] = config_dir
+    config["config_dir"] = config_dir
 
-    config_d = os.path.join(config['config_dir'], 'config.d')
-    sources_d = os.path.join(config['config_dir'], 'sources.d')
+    config_d = os.path.join(config["config_dir"], "config.d")
+    sources_d = os.path.join(config["config_dir"], "sources.d")
 
     # Merge in config.d/*.json to the root level
     config_files = []
@@ -114,7 +114,7 @@
         config_files = [
             os.path.join(config_d, fn)
             for fn in os.listdir(config_d)
-            if fn.endswith('.json')
+            if fn.endswith(".json")
             and os.path.isfile(os.path.join(config_d, fn))
             and os.access(os.path.join(config_d, fn), os.R_OK)
         ]
@@ -122,10 +122,10 @@
     for file in config_files:
         config = dict_merge(config, json_load_file(file))
 
-    if 'var_dir' not in config:
-        config['var_dir'] = '/var/lib/turku-agent'
+    if "var_dir" not in config:
+        config["var_dir"] = "/var/lib/turku-agent"
 
-    var_config_d = os.path.join(config['var_dir'], 'config.d')
+    var_config_d = os.path.join(config["var_dir"], "config.d")
 
     # Load /var config.d files
     var_config = {}
@@ -134,7 +134,7 @@
         var_config_files = [
             os.path.join(var_config_d, fn)
             for fn in os.listdir(var_config_d)
-            if fn.endswith('.json')
+            if fn.endswith(".json")
             and os.path.isfile(os.path.join(var_config_d, fn))
             and os.access(os.path.join(var_config_d, fn), os.R_OK)
         ]
@@ -145,40 +145,40 @@
     var_config = dict_merge(var_config, config)
     config = var_config
 
-    if 'lock_dir' not in config:
-        config['lock_dir'] = '/var/lock'
-
-    if 'rsyncd_command' not in config:
-        config['rsyncd_command'] = ['rsync']
-
-    if 'rsyncd_local_address' not in config:
-        config['rsyncd_local_address'] = '127.0.0.1'
-
-    if 'rsyncd_local_port' not in config:
-        config['rsyncd_local_port'] = 27873
-
-    if 'ssh_command' not in config:
-        config['ssh_command'] = ['ssh']
+    if "lock_dir" not in config:
+        config["lock_dir"] = "/var/lock"
+
+    if "rsyncd_command" not in config:
+        config["rsyncd_command"] = ["rsync"]
+
+    if "rsyncd_local_address" not in config:
+        config["rsyncd_local_address"] = "127.0.0.1"
+
+    if "rsyncd_local_port" not in config:
+        config["rsyncd_local_port"] = 27873
+
+    if "ssh_command" not in config:
+        config["ssh_command"] = ["ssh"]
 
     # If a go/no-go program is defined, run it and only go if it exits 0.
     # Type: String (program with no args) or list (program first, optional arguments after)
-    if 'gonogo_program' not in config:
-        config['gonogo_program'] = None
+    if "gonogo_program" not in config:
+        config["gonogo_program"] = None
 
-    var_sources_d = os.path.join(config['var_dir'], 'sources.d')
+    var_sources_d = os.path.join(config["var_dir"], "sources.d")
 
     # Validate the unit name
-    if 'unit_name' not in config:
-        config['unit_name'] = platform.node()
+    if "unit_name" not in config:
+        config["unit_name"] = platform.node()
         # If this isn't in the on-disk config, don't write it; just
         # generate it every time
 
     # Pull the SSH public key
-    if os.path.isfile(os.path.join(config['var_dir'], 'ssh_key.pub')):
-        with open(os.path.join(config['var_dir'], 'ssh_key.pub')) as f:
-            config['ssh_public_key'] = f.read().rstrip()
-        config['ssh_public_key_file'] = os.path.join(config['var_dir'], 'ssh_key.pub')
-        config['ssh_private_key_file'] = os.path.join(config['var_dir'], 'ssh_key')
+    if os.path.isfile(os.path.join(config["var_dir"], "ssh_key.pub")):
+        with open(os.path.join(config["var_dir"], "ssh_key.pub")) as f:
+            config["ssh_public_key"] = f.read().rstrip()
+        config["ssh_public_key_file"] = os.path.join(config["var_dir"], "ssh_key.pub")
+        config["ssh_private_key_file"] = os.path.join(config["var_dir"], "ssh_key")
 
     sources_config = {}
     # Merge in sources.d/*.json to the sources dict
@@ -187,7 +187,7 @@
         sources_files = [
             os.path.join(sources_d, fn)
             for fn in os.listdir(sources_d)
-            if fn.endswith('.json')
+            if fn.endswith(".json")
             and os.path.isfile(os.path.join(sources_d, fn))
             and os.access(os.path.join(sources_d, fn), os.R_OK)
         ]
@@ -197,7 +197,7 @@
         var_sources_files = [
             os.path.join(var_sources_d, fn)
             for fn in os.listdir(var_sources_d)
-            if fn.endswith('.json')
+            if fn.endswith(".json")
             and os.path.isfile(os.path.join(var_sources_d, fn))
             and os.access(os.path.join(var_sources_d, fn), os.R_OK)
         ]
@@ -208,19 +208,19 @@
 
     # Check for required sources options
     for s in list(sources_config.keys()):
-        if 'path' not in sources_config[s]:
+        if "path" not in sources_config[s]:
             del sources_config[s]
 
-    config['sources'] = sources_config
+    config["sources"] = sources_config
 
     return config
 
 
 def fill_config(config):
-    config_d = os.path.join(config['config_dir'], 'config.d')
-    sources_d = os.path.join(config['config_dir'], 'sources.d')
-    var_config_d = os.path.join(config['var_dir'], 'config.d')
-    var_sources_d = os.path.join(config['var_dir'], 'sources.d')
+    config_d = os.path.join(config["config_dir"], "config.d")
+    sources_d = os.path.join(config["config_dir"], "sources.d")
+    var_config_d = os.path.join(config["var_dir"], "config.d")
+    var_sources_d = os.path.join(config["var_dir"], "sources.d")
 
     # Create required directories
     for d in (config_d, sources_d, var_config_d, var_sources_d):
@@ -229,106 +229,122 @@
 
     # Validate the machine UUID/secret
     write_uuid_data = False
-    if 'machine_uuid' not in config:
-        config['machine_uuid'] = str(uuid.uuid4())
+    if "machine_uuid" not in config:
+        config["machine_uuid"] = str(uuid.uuid4())
         write_uuid_data = True
-    if 'machine_secret' not in config:
-        config['machine_secret'] = ''.join(
-            random.choice(string.ascii_letters + string.digits)
-            for i in range(30)
+    if "machine_secret" not in config:
+        config["machine_secret"] = "".join(
+            random.choice(string.ascii_letters + string.digits) for i in range(30)
         )
         write_uuid_data = True
     # Write out the machine UUID/secret if needed
     if write_uuid_data:
-        with open(os.path.join(var_config_d, '10-machine_uuid.json'), 'w') as f:
+        with open(os.path.join(var_config_d, "10-machine_uuid.json"), "w") as f:
             os.fchmod(f.fileno(), 0o600)
-            json_dump_p({
-                'machine_uuid': config['machine_uuid'],
-                'machine_secret': config['machine_secret'],
-            }, f)
+            json_dump_p(
+                {
+                    "machine_uuid": config["machine_uuid"],
+                    "machine_secret": config["machine_secret"],
+                },
+                f,
+            )
 
     # Restoration configuration
     write_restore_data = False
-    if 'restore_path' not in config:
-        config['restore_path'] = '/var/backups/turku-agent/restore'
-        write_restore_data = True
-    if 'restore_module' not in config:
-        config['restore_module'] = 'turku-restore'
-        write_restore_data = True
-    if 'restore_username' not in config:
-        config['restore_username'] = str(uuid.uuid4())
-        write_restore_data = True
-    if 'restore_password' not in config:
-        config['restore_password'] = ''.join(
-            random.choice(string.ascii_letters + string.digits)
-            for i in range(30)
+    if "restore_path" not in config:
+        config["restore_path"] = "/var/backups/turku-agent/restore"
+        write_restore_data = True
+    if "restore_module" not in config:
+        config["restore_module"] = "turku-restore"
+        write_restore_data = True
+    if "restore_username" not in config:
+        config["restore_username"] = str(uuid.uuid4())
+        write_restore_data = True
+    if "restore_password" not in config:
+        config["restore_password"] = "".join(
+            random.choice(string.ascii_letters + string.digits) for i in range(30)
         )
         write_restore_data = True
     if write_restore_data:
-        with open(os.path.join(var_config_d, '10-restore.json'), 'w') as f:
+        with open(os.path.join(var_config_d, "10-restore.json"), "w") as f:
             os.fchmod(f.fileno(), 0o600)
             restore_out = {
-                'restore_path': config['restore_path'],
-                'restore_module': config['restore_module'],
-                'restore_username': config['restore_username'],
-                'restore_password': config['restore_password'],
+                "restore_path": config["restore_path"],
+                "restore_module": config["restore_module"],
+                "restore_username": config["restore_username"],
+                "restore_password": config["restore_password"],
             }
             json_dump_p(restore_out, f)
-    if not os.path.isdir(config['restore_path']):
-        os.makedirs(config['restore_path'])
+    if not os.path.isdir(config["restore_path"]):
+        os.makedirs(config["restore_path"])
 
     # Generate the SSH keypair if it doesn't exist
-    if 'ssh_private_key_file' not in config:
-        subprocess.check_call([
-            'ssh-keygen', '-t', 'rsa', '-N', '', '-C', 'turku',
-            '-f', os.path.join(config['var_dir'], 'ssh_key')
-        ])
-        with open(os.path.join(config['var_dir'], 'ssh_key.pub')) as f:
-            config['ssh_public_key'] = f.read().rstrip()
-        config['ssh_public_key_file'] = os.path.join(config['var_dir'], 'ssh_key.pub')
-        config['ssh_private_key_file'] = os.path.join(config['var_dir'], 'ssh_key')
+    if "ssh_private_key_file" not in config:
+        subprocess.check_call(
+            [
+                "ssh-keygen",
+                "-t",
+                "rsa",
+                "-N",
+                "",
+                "-C",
+                "turku",
+                "-f",
+                os.path.join(config["var_dir"], "ssh_key"),
+            ]
+        )
+        with open(os.path.join(config["var_dir"], "ssh_key.pub")) as f:
+            config["ssh_public_key"] = f.read().rstrip()
+        config["ssh_public_key_file"] = os.path.join(config["var_dir"], "ssh_key.pub")
+        config["ssh_private_key_file"] = os.path.join(config["var_dir"], "ssh_key")
 
-    for s in config['sources']:
+    for s in config["sources"]:
         # Check for missing usernames/passwords
-        if not ('username' in config['sources'][s] or 'password' in config['sources'][s]):
-            sources_secrets_d = os.path.join(config['config_dir'], 'sources_secrets.d')
-            if 'username' not in config['sources'][s]:
-                config['sources'][s]['username'] = str(uuid.uuid4())
-            if 'password' not in config['sources'][s]:
-                config['sources'][s]['password'] = ''.join(
+        if not (
+            "username" in config["sources"][s] or "password" in config["sources"][s]
+        ):
+            if "username" not in config["sources"][s]:
+                config["sources"][s]["username"] = str(uuid.uuid4())
+            if "password" not in config["sources"][s]:
+                config["sources"][s]["password"] = "".join(
                     random.choice(string.ascii_letters + string.digits)
                     for i in range(30)
                 )
-            with open(os.path.join(var_sources_d, '10-' + s + '.json'), 'w') as f:
+            with open(os.path.join(var_sources_d, "10-" + s + ".json"), "w") as f:
                 os.fchmod(f.fileno(), 0o600)
-                json_dump_p({
-                    s: {
-                        'username': config['sources'][s]['username'],
-                        'password': config['sources'][s]['password'],
-                    }
-                }, f)
+                json_dump_p(
+                    {
+                        s: {
+                            "username": config["sources"][s]["username"],
+                            "password": config["sources"][s]["password"],
+                        }
+                    },
+                    f,
+                )
 
 
 def api_call(api_url, cmd, post_data, timeout=5):
     url = urllib.parse.urlparse(api_url)
-    if url.scheme == 'https':
+    if url.scheme == "https":
         h = http.client.HTTPSConnection(url.netloc, timeout=timeout)
     else:
         h = http.client.HTTPConnection(url.netloc, timeout=timeout)
     out = json.dumps(post_data)
-    h.putrequest('POST', '%s/%s' % (url.path, cmd))
-    h.putheader('Content-Type', 'application/json')
-    h.putheader('Content-Length', len(out))
-    h.putheader('Accept', 'application/json')
+    h.putrequest("POST", "%s/%s" % (url.path, cmd))
+    h.putheader("Content-Type", "application/json")
+    h.putheader("Content-Length", len(out))
+    h.putheader("Accept", "application/json")
     h.endheaders()
-    h.send(out.encode('UTF-8'))
+    h.send(out.encode("UTF-8"))
 
     res = h.getresponse()
     if not res.status == http.client.OK:
-        raise Exception('Received error %d (%s) from API server' % (res.status, res.reason))
-    if not res.getheader('content-type') == 'application/json':
-        raise Exception('Received invalid reply from API server')
+        raise Exception(
+            "Received error %d (%s) from API server" % (res.status, res.reason)
+        )
+    if not res.getheader("content-type") == "application/json":
+        raise Exception("Received invalid reply from API server")
     try:
-        return json.loads(res.read().decode('UTF-8'))
+        return json.loads(res.read().decode("UTF-8"))
     except ValueError:
-        raise Exception('Received invalid reply from API server')
+        raise Exception("Received invalid reply from API server")