Merge ~smoser/usd-importer:fix/1749609-build-non-1000 into usd-importer:master

Proposed by Scott Moser on 2018-09-10
Status: Merged
Merged at revision: 772811277a5bd7a02f8dbee719e53c6b20ed5221
Proposed branch: ~smoser/usd-importer:fix/1749609-build-non-1000
Merge into: usd-importer:master
Diff against target: 148 lines (+38/-52)
1 file modified
gitubuntu/build.py (+38/-52)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Approve on 2018-09-18
Karl Stenerud (community) 2018-09-12 Approve on 2018-09-13
Robie Basak 2018-09-10 Pending
Review via email: mp+354634@code.launchpad.net

Commit message

Fix 'git ubuntu build' when user id does not match container user id.

If the user running 'git ubuntu build' did not have the same user-id
as the user 'ubuntu' inside the container, then the build would fail.
This fixes the specific issue.

The changes along the way are:
a.) builds as 'build-user' rather than 'ubuntu'.
    A consistent distro-nuetral user just seems to make more sense.
b.) build in /home/build-user/buildd rather than /tmp/.
    This seems cleaner as other no other system processes will be
    adding or removing files from that directory.

LP: #1749609

Description of the change

see commit message

To post a comment you must log in.

FAILED: Continuous integration, rev:fa40cb0db81680d20e2b48401797c90489c59bea
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/25/
Executed test runs:
    FAILED: VM Setup

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/25/rebuild

review: Needs Fixing (continuous-integration)
Joshua Powers (powersj) wrote :

Ignore the bot, he is still not 100% working due to proxy issues.

PASSED: Continuous integration, rev:fa40cb0db81680d20e2b48401797c90489c59bea
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/27/
Executed test runs:
    SUCCESS: VM Setup
    SUCCESS: Build
    SUCCESS: Unit Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/27/rebuild

review: Approve (continuous-integration)
Scott Moser (smoser) wrote :

@Karl,
Since you reported/re-reported this issue, I wonder if you could test to see if this fixes your problem.

Heres how to test:

 $ git clone https://git.launchpad.net/~smoser/usd-importer
 $ cd usd-importer
 $ git checkout fix/1749609-build-non-1000
 $ PATH=$PWD/bin:$PATH
 $ cd ..
 $ git ubuntu clone postfix
 $ cd postfix
 $ git ubuntu build --verbose --lxd-image=ubuntu-daily:bionic

We currently need to tell it to build in bionic (--lxd-image=ubuntu-daily:bionic) just because of bug 1791691.

I've done the above test locally, but it'd be good to make sure it also fixes the problem for you.

I *have* gone through the above

Karl Stenerud (kstenerud) wrote :

Works for me! Thanks!

review: Approve

PASSED: Continuous integration, rev:772811277a5bd7a02f8dbee719e53c6b20ed5221
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/29/
Executed test runs:
    SUCCESS: VM Setup
    SUCCESS: Build
    SUCCESS: Unit Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/29/rebuild

review: Approve (continuous-integration)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/gitubuntu/build.py b/gitubuntu/build.py
2index b7ea657..5e892e0 100644
3--- a/gitubuntu/build.py
4+++ b/gitubuntu/build.py
5@@ -957,6 +957,8 @@ def do_build_lxd_exitstack(
6 raise RuntimeError("LXD running?") from e
7
8 container_name = petname.Generate(2, '-')
9+ user = 'buildd'
10+ buildd = "/home/%s/build" % user
11
12 args = ['launch', '-e']
13 if profile:
14@@ -1019,61 +1021,50 @@ def do_build_lxd_exitstack(
15 "Failed to run apt-get in ephemeral build container"
16 )
17
18+ input_files = [archive_tarball_name] + tarballs
19+ try:
20+ _run_in_lxd(container_name, ['mkdir', '-p', buildd], user=user)
21+ except Exception as e:
22+ raise RuntimeError(
23+ "Failed creating '%s' as user '%s' in container '%s'" %
24+ (buildd, user, container_name)) from e
25+
26+ logging.debug("container=%s build_dir=%s user=%s",
27+ container_name, buildd, user)
28+
29 try:
30 run_lxc(
31- [
32- 'file',
33- 'push',
34- archive_tarball_name,
35- '%s/tmp/' % container_name,
36- ],
37- )
38+ ['file', 'push'] + input_files +
39+ ['%s/%s/' % (container_name, buildd)])
40 except Exception as e:
41 raise RuntimeError(
42 "Failed to push archive tarball to ephemeral build container"
43 ) from e
44
45- args = ['file', 'push']
46- args.extend(tarballs)
47- args.extend(['%s/tmp/' % container_name,])
48- try:
49- run_lxc(args)
50- except Exception as e:
51- raise RuntimeError("Failed to push orig tarballs to container") from e
52+ logging.info(
53+ "Copied build files to %s %s: %s",
54+ container_name, buildd,
55+ [os.path.basename(f) for f in input_files])
56
57- logging.info("Copied build files to %s", container_name)
58+ _run_in_lxd(container_name, ['chown', '--recursive', user, buildd])
59
60 try:
61- _run_in_lxd(
62+ _run_in_lxd(
63 container_name,
64 [
65 'tar',
66 'zxCf',
67- '/tmp',
68- '/tmp/%s' % os.path.basename(archive_tarball_name),
69+ buildd,
70+ '%s/%s' % (buildd, os.path.basename(archive_tarball_name)),
71 ],
72- user='ubuntu',
73+ user=user,
74 )
75 except Exception as e:
76 raise RuntimeError(
77 "Failed to untar archive tarball in container"
78 ) from e
79
80- stdout, _ = _run_in_lxd(
81- container_name,
82- ['ls', '/tmp',],
83- user='ubuntu',
84- )
85- orig_temp_contents = set(
86- filename_stripped
87- for filename_stripped in (
88- filename.strip() for filename in
89- stdout.splitlines()
90- )
91- if filename_stripped
92- )
93-
94- container_path = '/tmp/%s-%s' % (
95+ container_path = buildd + '/%s-%s' % (
96 changelog.srcpkg,
97 changelog.version,
98 )
99@@ -1113,36 +1104,31 @@ def do_build_lxd_exitstack(
100 ' '.join(rem_args),
101 ),
102 ],
103- user='ubuntu',
104+ user=user,
105 )
106 except Exception as e:
107 raise RuntimeError("dpkg-buildpackage failed in the container") from e
108
109- stdout, _ = _run_in_lxd(
110- container_name,
111- ['ls', '/tmp',],
112- user='ubuntu',
113- )
114- new_temp_contents = set(
115- filename_stripped
116- for filename_stripped in (
117- filename.strip() for filename in
118- stdout.splitlines()
119- )
120- if filename_stripped
121- )
122+ ls_out, _ = _run_in_lxd(
123+ container_name, ['ls', buildd], user=user)
124+ new_contents = [f for f in ls_out.splitlines() if f]
125+ built_contents = (
126+ set(new_contents) - set([os.path.basename(i) for i in input_files]) -
127+ set((os.path.basename(container_path),)))
128+ if not built_contents:
129+ raise RuntimeError("No new files in build dir. build failed?")
130+ logging.debug("Build produced files: %s" % built_contents)
131
132- built_temp_contents = new_temp_contents - orig_temp_contents
133 args = ['file', 'pull']
134 args.extend([
135- '%s%s' % (container_name, os.path.join('/tmp', f))
136- for f in built_temp_contents
137+ '%s%s' % (container_name, os.path.join(buildd, f))
138+ for f in built_contents
139 ]
140 )
141 args.extend([os.pardir,])
142 run_lxc(args)
143
144- return [os.path.join(os.pardir, f) for f in built_temp_contents]
145+ return [os.path.join(os.pardir, f) for f in built_contents]
146
147
148 def fetch_orig(

Subscribers

People subscribed via source and target branches