Merge autopkgtest-cloud:wip/private into autopkgtest-cloud:master

[Iain Lane]

This is a WIP / RFC. Comments welcome from autopkgtest developers and potential users of this (security team?)

I've implemented some initial parts for running private jobs. If "private" is set to true in the parameters, then the following things will happen

  - The test will be displayed in the running page (both queue and the log tail) as 'private'
  - It'll be put a container prefixed by 'private-'

I don't know how to handle access control for the private containers. Currently nobody except our own tenant can access the containers. Comments would be welcome on that subject. One thing Swift supports is access control based on Openstack tenant, so consumers could run their own code to download the results themselves (Q: how would we map container to tenant?). Presumably if they're handling job triggering then they can handle collecting results too.

Alternatively autopkgtest-web could be extended to do this somehow, via some kind of proxying of the objects in swift, together with appropriate authentication which I really don't know the details of either.

If you have access to the infrastructure, then: to see a couple of test runs, check out "journalctl ADT_PACKAGE=gzip" from today (2019-04-05), and the 'private-autopkgtest-disco' container in our swift.

[Iain Lane (laney)]

This is superseded by https://code.launchpad.net/~ubuntu-release/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/399668

review: Needs Resubmitting

Unmerged commits

9e17409... by Iain Lane on 2019-04-03

WIP: worker: Support 'private' tests

If private is passed as a parameter, don't send status updates and hide
queue items.

TODO: How to receive requests from a private PPA?
      Make private PPAs automatically imply a private job?
      ACL for the resulting swift container (then how are the results
      gathered? up to the consumer?)

diff --git a/webcontrol/browse.cgi b/webcontrol/browse.cgi
index 85e4b4f..116596a 100755
--- a/webcontrol/browse.cgi
+++ b/webcontrol/browse.cgi
@@ -127,7 +127,18 @@ def get_queue_requests(amqp_channel, queue_name):
     for r in requests:
         if isinstance(r, bytes):
             r = r.decode('UTF-8')
-        res.append(r)
+        try:
+            req = r.split(None, 1)
+            if len(req) > 1:
+                params = json.loads(req[1])
+            else:
+                params = {}
+            if params.get('private', False):
+                r = 'private job'
+            res.append(r)
+        except (ValueError, IndexError):
+            logging.error('Received invalid request format "%s"', r)
+            return
     return res
 
 
diff --git a/webcontrol/request/submit.py b/webcontrol/request/submit.py
index 31eddd5..f92e470 100644
--- a/webcontrol/request/submit.py
+++ b/webcontrol/request/submit.py
@@ -79,6 +79,12 @@ class Submit:
             del kwargs['all-proposed']
         except KeyError:
             pass
+        try:
+            if kwargs['private'] != '1':
+                raise ValueError('Invalid private value')
+            del kwargs['private']
+        except KeyError:
+            pass
         # no other kwargs supported
         if kwargs:
             raise ValueError('Invalid argument %s' % list(kwargs)[0])
diff --git a/worker/worker b/worker/worker
index f431a6c..bb5be28 100755
--- a/worker/worker
+++ b/worker/worker
@@ -190,26 +190,31 @@ def subst(s, autopkgtest_checkout, big_package, release, architecture, pkgname):
     return s
 
 
-def send_status_info(queue, release, architecture, pkgname, params, out_dir, running, duration):
+def send_status_info(queue, release, architecture, pkgname, params, out_dir, running, duration, private=False):
     '''Send status and logtail to status queue'''
 
     if not queue:
         return
 
-    # print('status_info:', release, architecture, pkgname, out_dir, running)
-    try:
-        with open(os.path.join(out_dir, 'log'), 'rb') as f:
-            try:
-                f.seek(-2000, os.SEEK_END)
-                # throw away the first line as we almost surely cut that out in
-                # the middle
-                f.readline()
-            except IOError:
-                # file is smaller than 2000 bytes? okay
-                pass
-            logtail = f.read().decode('UTF-8', errors='replace')
-    except (IOError, OSError) as e:
-        logtail = 'Cannot read log file: %s' % e
+    if private:
+        pkgname = 'private-test'
+        params = {}
+        logtail = 'Running private test'
+    else:
+        # print('status_info:', release, architecture, pkgname, out_dir, running)
+        try:
+            with open(os.path.join(out_dir, 'log'), 'rb') as f:
+                try:
+                    f.seek(-2000, os.SEEK_END)
+                    # throw away the first line as we almost surely cut that out in
+                    # the middle
+                    f.readline()
+                except IOError:
+                    # file is smaller than 2000 bytes? okay
+                    pass
+                logtail = f.read().decode('UTF-8', errors='replace')
+        except (IOError, OSError) as e:
+            logtail = 'Cannot read log file: %s' % e
 
     msg = json.dumps({'release': release,
                       'architecture': architecture,
@@ -220,8 +225,7 @@ def send_status_info(queue, release, architecture, pkgname, params, out_dir, run
                       'logtail': logtail})
     queue.basic_publish(amqp.Message(msg, delivery_mode=2), status_exchange_name, '')
 
-
-def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time):
+def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time, private=False):
     '''Call autopkgtest and regularly send status/logtail to status_exchange_name
 
     Return exit code.
@@ -242,11 +246,13 @@ def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, star
         status_update_counter = (status_update_counter + 1) % 10
         if status_update_counter == 0:
             send_status_info(status_amqp, release, architecture, pkgname,
-                             params, out_dir, True, int(time.time() - start_time))
+                             params, out_dir, True, int(time.time() - start_time),
+                             private)
 
     ret = autopkgtest.wait()
     send_status_info(status_amqp, release, architecture, pkgname, params,
-                     out_dir, False, int(time.time() - start_time))
+                     out_dir, False, int(time.time() - start_time),
+                     private)
 
     return ret
 
@@ -478,6 +484,11 @@ def request(msg):
     argv += subst(cfg.get('virt', 'args'), autopkgtest_checkout, big_pkg,
                   release, architecture, pkgname).split()
 
+    private = params.get('private', False)
+
+    if private:
+        container = 'private-{}'.format(container)
+
     # run autopkgtest; retry up to three times on tmpfail issues
     if not blacklisted:
         global running_test
@@ -487,7 +498,8 @@ def request(msg):
         for retry in range(3):
             retry_start_time = time.time()
             logging.info('Running %s', ' '.join(argv))
-            code = call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time)
+            code = call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time,
+                                    private)
             if code == 16 or code < 0:
                 if exit_requested is not None:
                     logging.warning('Testbed failure and exit %i requested. Log follows:', exit_requested)
@@ -588,8 +600,16 @@ def request(msg):
         swift_con.get_container(container, limit=1)
     except swiftclient.exceptions.ClientException:
         logging.info('container %s does not exist, creating it', container)
-        # make it publicly readable
-        swift_con.put_container(container, headers={'X-Container-Read': '.rlistings,.r:*'})
+        if not private:
+            # make it publicly readable
+            swift_con.put_container(container, headers={'X-Container-Read': '.rlistings,.r:*'})
+        else:
+            # XXX: make this have some kind of good access control. like one
+            # of:
+            # - openstack tenant. consumers will use our $OS_STORAGE_URL and
+            #   then be able to retrieve the swift objects directly
+            # - IP range (e.g. Canonical network)
+            swift_con.put_container(container)
         # wait until it exists
         timeout = 50
         while timeout > 0: