Merge autopkgtest-cloud:wip/private into autopkgtest-cloud:master

Proposed by Iain Lane on 2019-04-05
Status: Needs review
Proposed branch: autopkgtest-cloud:wip/private
Merge into: autopkgtest-cloud:master
Diff against target: 159 lines (+60/-23)
3 files modified
webcontrol/browse.cgi (+12/-1)
webcontrol/request/submit.py (+6/-0)
worker/worker (+42/-22)
Reviewer Review Type Date Requested Status
Steve Langasek 2019-04-05 Pending
Ubuntu Release Team 2019-04-05 Pending
Review via email: mp+365586@code.launchpad.net

Description of the change

This is a WIP / RFC. Comments welcome from autopkgtest developers and potential users of this (security team?)

I've implemented some initial parts for running private jobs. If "private" is set to true in the parameters, then the following things will happen

  - The test will be displayed in the running page (both queue and the log tail) as 'private'
  - It'll be put a container prefixed by 'private-'

I don't know how to handle access control for the private containers. Currently nobody except our own tenant can access the containers. Comments would be welcome on that subject. One thing Swift supports is access control based on Openstack tenant, so consumers could run their own code to download the results themselves (Q: how would we map container to tenant?). Presumably if they're handling job triggering then they can handle collecting results too.

Alternatively autopkgtest-web could be extended to do this somehow, via some kind of proxying of the objects in swift, together with appropriate authentication which I really don't know the details of either.

If you have access to the infrastructure, then: to see a couple of test runs, check out "journalctl ADT_PACKAGE=gzip" from today (2019-04-05), and the 'private-autopkgtest-disco' container in our swift.

To post a comment you must log in.
Iain Lane (laney) :

Unmerged commits

9e17409... by Iain Lane on 2019-04-03

WIP: worker: Support 'private' tests

If private is passed as a parameter, don't send status updates and hide
queue items.

TODO: How to receive requests from a private PPA?
      Make private PPAs automatically imply a private job?
      ACL for the resulting swift container (then how are the results
      gathered? up to the consumer?)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/webcontrol/browse.cgi b/webcontrol/browse.cgi
2index 85e4b4f..116596a 100755
3--- a/webcontrol/browse.cgi
4+++ b/webcontrol/browse.cgi
5@@ -127,7 +127,18 @@ def get_queue_requests(amqp_channel, queue_name):
6 for r in requests:
7 if isinstance(r, bytes):
8 r = r.decode('UTF-8')
9- res.append(r)
10+ try:
11+ req = r.split(None, 1)
12+ if len(req) > 1:
13+ params = json.loads(req[1])
14+ else:
15+ params = {}
16+ if params.get('private', False):
17+ r = 'private job'
18+ res.append(r)
19+ except (ValueError, IndexError):
20+ logging.error('Received invalid request format "%s"', r)
21+ return
22 return res
23
24
25diff --git a/webcontrol/request/submit.py b/webcontrol/request/submit.py
26index 31eddd5..f92e470 100644
27--- a/webcontrol/request/submit.py
28+++ b/webcontrol/request/submit.py
29@@ -79,6 +79,12 @@ class Submit:
30 del kwargs['all-proposed']
31 except KeyError:
32 pass
33+ try:
34+ if kwargs['private'] != '1':
35+ raise ValueError('Invalid private value')
36+ del kwargs['private']
37+ except KeyError:
38+ pass
39 # no other kwargs supported
40 if kwargs:
41 raise ValueError('Invalid argument %s' % list(kwargs)[0])
42diff --git a/worker/worker b/worker/worker
43index f431a6c..bb5be28 100755
44--- a/worker/worker
45+++ b/worker/worker
46@@ -190,26 +190,31 @@ def subst(s, autopkgtest_checkout, big_package, release, architecture, pkgname):
47 return s
48
49
50-def send_status_info(queue, release, architecture, pkgname, params, out_dir, running, duration):
51+def send_status_info(queue, release, architecture, pkgname, params, out_dir, running, duration, private=False):
52 '''Send status and logtail to status queue'''
53
54 if not queue:
55 return
56
57- # print('status_info:', release, architecture, pkgname, out_dir, running)
58- try:
59- with open(os.path.join(out_dir, 'log'), 'rb') as f:
60- try:
61- f.seek(-2000, os.SEEK_END)
62- # throw away the first line as we almost surely cut that out in
63- # the middle
64- f.readline()
65- except IOError:
66- # file is smaller than 2000 bytes? okay
67- pass
68- logtail = f.read().decode('UTF-8', errors='replace')
69- except (IOError, OSError) as e:
70- logtail = 'Cannot read log file: %s' % e
71+ if private:
72+ pkgname = 'private-test'
73+ params = {}
74+ logtail = 'Running private test'
75+ else:
76+ # print('status_info:', release, architecture, pkgname, out_dir, running)
77+ try:
78+ with open(os.path.join(out_dir, 'log'), 'rb') as f:
79+ try:
80+ f.seek(-2000, os.SEEK_END)
81+ # throw away the first line as we almost surely cut that out in
82+ # the middle
83+ f.readline()
84+ except IOError:
85+ # file is smaller than 2000 bytes? okay
86+ pass
87+ logtail = f.read().decode('UTF-8', errors='replace')
88+ except (IOError, OSError) as e:
89+ logtail = 'Cannot read log file: %s' % e
90
91 msg = json.dumps({'release': release,
92 'architecture': architecture,
93@@ -220,8 +225,7 @@ def send_status_info(queue, release, architecture, pkgname, params, out_dir, run
94 'logtail': logtail})
95 queue.basic_publish(amqp.Message(msg, delivery_mode=2), status_exchange_name, '')
96
97-
98-def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time):
99+def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time, private=False):
100 '''Call autopkgtest and regularly send status/logtail to status_exchange_name
101
102 Return exit code.
103@@ -242,11 +246,13 @@ def call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, star
104 status_update_counter = (status_update_counter + 1) % 10
105 if status_update_counter == 0:
106 send_status_info(status_amqp, release, architecture, pkgname,
107- params, out_dir, True, int(time.time() - start_time))
108+ params, out_dir, True, int(time.time() - start_time),
109+ private)
110
111 ret = autopkgtest.wait()
112 send_status_info(status_amqp, release, architecture, pkgname, params,
113- out_dir, False, int(time.time() - start_time))
114+ out_dir, False, int(time.time() - start_time),
115+ private)
116
117 return ret
118
119@@ -478,6 +484,11 @@ def request(msg):
120 argv += subst(cfg.get('virt', 'args'), autopkgtest_checkout, big_pkg,
121 release, architecture, pkgname).split()
122
123+ private = params.get('private', False)
124+
125+ if private:
126+ container = 'private-{}'.format(container)
127+
128 # run autopkgtest; retry up to three times on tmpfail issues
129 if not blacklisted:
130 global running_test
131@@ -487,7 +498,8 @@ def request(msg):
132 for retry in range(3):
133 retry_start_time = time.time()
134 logging.info('Running %s', ' '.join(argv))
135- code = call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time)
136+ code = call_autopkgtest(argv, release, architecture, pkgname, params, out_dir, start_time,
137+ private)
138 if code == 16 or code < 0:
139 if exit_requested is not None:
140 logging.warning('Testbed failure and exit %i requested. Log follows:', exit_requested)
141@@ -588,8 +600,16 @@ def request(msg):
142 swift_con.get_container(container, limit=1)
143 except swiftclient.exceptions.ClientException:
144 logging.info('container %s does not exist, creating it', container)
145- # make it publicly readable
146- swift_con.put_container(container, headers={'X-Container-Read': '.rlistings,.r:*'})
147+ if not private:
148+ # make it publicly readable
149+ swift_con.put_container(container, headers={'X-Container-Read': '.rlistings,.r:*'})
150+ else:
151+ # XXX: make this have some kind of good access control. like one
152+ # of:
153+ # - openstack tenant. consumers will use our $OS_STORAGE_URL and
154+ # then be able to retrieve the swift objects directly
155+ # - IP range (e.g. Canonical network)
156+ swift_con.put_container(container)
157 # wait until it exists
158 timeout = 50
159 while timeout > 0:

Subscribers

People subscribed via source and target branches