Merge lp:~nskaggs/phablet-tools/add-lp-branch-support into lp:phablet-tools

Proposed by Nicholas Skaggs
Status: Work in progress
Proposed branch: lp:~nskaggs/phablet-tools/add-lp-branch-support
Merge into: lp:phablet-tools
Diff against target: 392 lines (+316/-18)
3 files modified
click-build.py (+231/-0)
phablet-click-test-setup (+61/-18)
po2mo.sh (+24/-0)
To merge this branch: bzr merge lp:~nskaggs/phablet-tools/add-lp-branch-support
Reviewer Review Type Date Requested Status
Ubuntu Phablet Team Pending
Review via email: mp+201685@code.launchpad.net

Commit message

Make some tweaks to phablet-click-test-setup to allow an easier workflow for app developers.

Add option to not bootstrap device everytime (we shouldn't need to constantly pull unity8 tests)
Add option to build click app and extract tests to target device for running
Add -c option (same as --click)

Description of the change

Make some tweaks to phablet-click-test-setup to allow an easier workflow for app developers. Desired workflow.

Hack on tests
Push to launchpad
Run tests on device / emulator via
phablet-click-test-setup lp:mybranch
phablet-test-run my_apptest

Add option to not bootstrap device everytime (we shouldn't need to constantly pull unity8 tests)
Add option to build click app and extract tests to target device for running
Add -c option (same as --click)

To post a comment you must log in.
239. By Nicholas Skaggs

remove old shell script

Revision history for this message
Nicholas Skaggs (nskaggs) wrote :

Note click_build.py is a tweaked version of Sergio's script. Need to migrate po2mo.sh functionality into click_build.py.

Revision history for this message
Sergio Schvezov (sergiusens) wrote :

Hm, I did this as an intermediate step until we got the debian packaging out of the way and all the path hardcoding was gone, that hasn't happened though; I would much rather have a simple cmake rules (or plain Makefile) instead of this.

This also doesn't work for compiled apps (such as notes, gallery, camera, mediaplayer, etc).

Unmerged revisions

239. By Nicholas Skaggs

remove old shell script

238. By Nicholas Skaggs

removed hacky globals, tweaked variable names

237. By Nicholas Skaggs

streamlined to make program more logical

236. By Nicholas Skaggs

fixed some bugs, hacky, but runs as expected

235. By Nicholas Skaggs

wip commit to move install_from_branch to click-test-setup

234. By Nicholas Skaggs

working version, nice and hacky

233. By Nicholas Skaggs

initial version of installing from an arbitrary lp branch to device

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== added file 'click-build.py'
--- click-build.py 1970-01-01 00:00:00 +0000
+++ click-build.py 2014-01-14 22:09:32 +0000
@@ -0,0 +1,231 @@
1#!/usr/bin/python3
2
3import argparse
4import atexit
5import configparser
6import json
7import os
8import shutil
9import sys
10import tempfile
11
12from subprocess import check_call, check_output
13
14
15tool_path = os.path.dirname(os.path.abspath(sys.modules['__main__'].__file__))
16
17
18def get_parser():
19 parser = argparse.ArgumentParser(
20 description='''Parses sources and stages them to build click
21 packages out of them.''')
22 parser.add_argument('--source-directory', dest='path',
23 default=os.path.curdir)
24 parser.add_argument('--bzr-source', required=True)
25 parser.add_argument('--bzr-revno', default=None)
26 parser.add_argument('--no-clean', action='store_true')
27 return parser
28
29
30def fix_exec_inline(exec_line, path):
31 print('Fixing exec inline %s with %s' % (exec_line, path))
32 exec_line_len = len(exec_line)
33 if exec_line_len == 3:
34 exec_line = '%s %s %s' % (
35 exec_line[0], strip_path(exec_line[1], path),
36 strip_path(exec_line[2], path))
37 elif exec_line_len == 2:
38 exec_line = '%s %s' % (exec_line[0], strip_path(exec_line[1], path))
39 elif exec_line_len == 1:
40 exec_line = strip_path(exec_line[0], path)
41 else:
42 raise NotImplementedError('Cannot handle Exec of len %s' %
43 exec_line_len)
44 print('New exec line is %s' % exec_line)
45 return exec_line
46
47
48def add_import(exec_line):
49 print('Adding import %s' % exec_line)
50 if 'qmlscene' in exec_line:
51 exec_line = '%s -I ./plugins' % exec_line
52 return exec_line
53
54
55def strip_path(item, path):
56 """Returns unstripped path for item."""
57 basename = os.path.basename(item)
58 if item == basename:
59 print('%s is good' % item)
60 else:
61 for root, sub, files in os.walk(path):
62 if basename in files:
63 basename = os.path.join(os.path.relpath(root, path), basename)
64 break
65 print('Replacing %s with %s' % (item, basename))
66 return basename
67
68
69def remove_cruft(path):
70 print('Checking for unneeded directories')
71 for cruft_dir in ('debian', 'tests', 'po'):
72 cruft_dir = os.path.join(path, cruft_dir)
73 if os.path.exists(cruft_dir):
74 print('Removing %s' % cruft_dir)
75 shutil.rmtree(cruft_dir)
76
77
78def get_plugins(path):
79 print('Looking for needed plugins')
80 plugin_manifest = os.path.join(path, 'plugins.json')
81 if not os.path.exists(plugin_manifest):
82 print('No extra plugins required')
83 return
84 plugin_target_path = os.path.join(path, 'plugins')
85 os.makedirs(plugin_target_path)
86 with open(plugin_manifest) as f:
87 manifest = json.loads(f.read())
88 staging_path = tempfile.mkdtemp()
89 atexit.register(lambda x: shutil.rmtree(x), staging_path)
90 for item in manifest:
91 exec_line = [os.path.join(tool_path, 'pull-lp-bin.py'),
92 '-o', staging_path, item['package']]
93 if 'ppa' in item:
94 exec_line += ['--ppa', item['ppa']]
95 check_call(exec_line, cwd=tool_path)
96 debs = [os.path.join(staging_path, x) for x in os.listdir(staging_path)
97 if x.endswith('.deb')]
98 for deb in debs:
99 extract_path = deb.strip('.deb')
100 check_call(['dpkg-deb', '-x', deb, extract_path])
101 plugin_path = os.path.join(extract_path,
102 'usr/lib/arm-linux-gnueabihf/qt5/qml')
103 for plugin in os.listdir(plugin_path):
104 print('Moving %s to %s' % (plugin, plugin_target_path))
105 shutil.move(os.path.join(plugin_path, plugin),
106 plugin_target_path)
107
108
109def run_translations(path, appname):
110 #trans_arg = [os.path.join(tool_path, 'po2mo.sh'), path, appname]
111 trans_arg = [os.path.join(tool_path, 'po2mo.sh'), path, appname, source_path]
112 print('Checking for translations with %s' % trans_arg)
113 check_call(trans_arg)
114
115
116class Click(object):
117
118 @property
119 def desktop(self):
120 for i in self.manifest['hooks']:
121 desktop = self.manifest['hooks'][i]['desktop']
122 print(desktop)
123 return desktop
124
125 @property
126 def name(self):
127 return self.manifest['name']
128
129 @property
130 def architecture(self):
131 return self.manifest['architecture'] \
132 if 'architecture' in self.manifest else None
133
134 def __init__(self, path):
135 self.manifest_path = os.path.join(path, 'manifest.json')
136 self.manifest_read()
137
138 def manifest_read(self):
139 print('Reading manifest.json')
140 with open(self.manifest_path) as f:
141 self.manifest = json.loads(f.read())
142
143 def manifest_write(self):
144 print('Writing manifest.json')
145 with open(self.manifest_path, 'w') as f:
146 for line in json.dumps(self.manifest, sort_keys=True, indent=4):
147 f.write(line)
148
149 def add_x_source(self, bzr_source, bzr_revno):
150 print('Adding source to manifest %s, at rev %s' % (bzr_source, bzr_revno))
151 self.manifest['x-source'] = {
152 'vcs-bzr': bzr_source,
153 'vcs-bzr-revno': bzr_revno
154 }
155
156 def append_version(self, bzr_revno):
157 print('Adding version to manifest %s' % (bzr_revno))
158 self.manifest['version'] += '.%s' % bzr_revno
159
160
161class Desktop(object):
162
163 def __init__(self, desktop_path):
164 self.desktop_path = desktop_path
165 self.read()
166
167 def read(self):
168 self.desktop = configparser.RawConfigParser()
169 self.desktop.optionxform = str
170 self.desktop.read(self.desktop_path)
171
172 def fix_icon(self, path):
173 print('Fixing icon %s' % path)
174 icon = self.desktop.get('Desktop Entry', 'Icon')
175 self.desktop.set('Desktop Entry', 'Icon', strip_path(icon, path))
176
177 def fix_exec(self, path, arch=None):
178 print('Fixing exec %s' % path)
179 exec_line = self.desktop.get('Desktop Entry', 'Exec').split()
180 exec_line = fix_exec_inline(exec_line, path)
181 if arch:
182 print('Creating custom exec')
183 exec_line = add_import(exec_line)
184 self.desktop.set('Desktop Entry', 'Exec', exec_line)
185
186 def write(self):
187 with open(self.desktop_path, 'w') as f:
188 self.desktop.write(f, space_around_delimiters=False)
189
190
191def setup_workdir(path, no_clean):
192 print('Setting up workdir %s' % path)
193 tempdir = tempfile.mkdtemp()
194 if not no_clean:
195 print('Cleaning up after work %s' % tempdir)
196 atexit.register(lambda x: shutil.rmtree(x), tempdir)
197 if not os.path.exists(path):
198 raise RuntimeError('Directory %s does not exist' % path)
199 source_path = os.path.join(tempdir, 'source')
200 check_call(['bzr', 'export', source_path], cwd=path)
201 print('Exported source to %s' % source_path)
202 return source_path
203
204
205def revno(path):
206 revno = check_output(['bzr', 'revno'], cwd=path).decode("utf-8").strip()
207 return revno
208
209
210if __name__ == '__main__':
211 parser = get_parser()
212 args = parser.parse_args()
213 path = os.path.abspath(args.path)
214 source_path = setup_workdir(path, args.no_clean)
215 click = Click(source_path)
216 bzr_revno = args.bzr_revno if args.bzr_revno else revno(path)
217 print('Building package from %s with revno %s' % (path, bzr_revno))
218 click.add_x_source(args.bzr_source, bzr_revno)
219 click.append_version(bzr_revno)
220 get_plugins(source_path)
221 desktop = Desktop(os.path.join(source_path, click.desktop))
222 desktop.fix_icon(source_path)
223 desktop.fix_exec(source_path, arch=click.architecture)
224 desktop.write()
225 click.manifest_write()
226 run_translations(source_path, click.name)
227 remove_cruft(source_path)
228 click_arg = ['click', 'build', source_path]
229 print('Building click package with %s' % click_arg)
230 check_call(click_arg)
231 print('Finished building click package from %s' % path)
0232
=== modified file 'phablet-click-test-setup'
--- phablet-click-test-setup 2013-10-21 18:14:51 +0000
+++ phablet-click-test-setup 2014-01-14 22:09:32 +0000
@@ -26,6 +26,7 @@
26import atexit26import atexit
27import json27import json
28import os28import os
29import sys
29import shutil30import shutil
30import tempfile31import tempfile
31import urllib232import urllib2
@@ -92,14 +93,29 @@
92 help='''Device serial. Use when more than93 help='''Device serial. Use when more than
93 one device is connected.''',94 one device is connected.''',
94 )95 )
95 parser.add_argument('--click', default=None,96 parser.add_argument('-c', '--click', default=None,
96 help='Specific click package to setup for.'97 help='Specific click package to setup for.'
97 )98 )
99 parser.add_argument('-b', '--branch', default=None,
100 help='Setup click package from lp branch. Requires \
101 you specify click package --click'
102 )
103 parser.add_argument('--no-bootstrap', action="store_false",
104 dest="bootstrap",
105 help='Do not setup basic packages '
106 )
98 parser.add_argument('--user', default='phablet',107 parser.add_argument('--user', default='phablet',
99 help='User on device to use')108 help='User on device to use')
100 parser.add_argument('--wipe', action='store_true',109 parser.add_argument('--wipe', action='store_true',
101 help='Clean up previous setup on device')110 help='Clean up previous setup on device')
102 return parser.parse_args()111
112 args = parser.parse_args()
113
114 #check dependency
115 if args.branch and not args.click:
116 parser.print_help()
117 sys.exit()
118 return args
103119
104120
105def cleanup(directory):121def cleanup(directory):
@@ -176,7 +192,7 @@
176 test_dir = path.join(test_base_dir, test_dir)192 test_dir = path.join(test_base_dir, test_dir)
177 print('Moving %s to %s' % (test_dir, target_dir))193 print('Moving %s to %s' % (test_dir, target_dir))
178 shutil.move(test_dir, target_dir)194 shutil.move(test_dir, target_dir)
179195 return tmp_dir
180196
181def fetch_test_base(adb, test_dir):197def fetch_test_base(adb, test_dir):
182 for package in python_packages:198 for package in python_packages:
@@ -185,32 +201,59 @@
185 version = adb.get_package_version(package['binary'])201 version = adb.get_package_version(package['binary'])
186 get_source_package_tests(package['source'], version, test_dir)202 get_source_package_tests(package['source'], version, test_dir)
187203
188204def fetch_click_tests(adb, test_dir, user, click=None, branch=None):
189def fetch_click_tests(adb, test_dir, user, click=None):205 if branch:
190 manifest = adb.get_click_manifest(user)206 #do we need to build and install the click package to the device?
191 if click:
192 print('Only setting up for %s' % click)207 print('Only setting up for %s' % click)
193 manifest = [entry for entry in manifest if click == entry['name']]208 print("Building and installing click package for branch %s" % branch)
194 print('Only keeping entries with x-source')209 #force checkout the latest version
195 manifest = [entry for entry in manifest if 'x-source' in entry]210 revision = "revno:-1"
196 for entry in manifest:211 tmp_dir=get_bzr_tests(branch, revision, test_dir)
197 get_bzr_tests(entry['x-source']['vcs-bzr'],212 install_click_from_branch(branch,
198 entry['x-source']['vcs-bzr-revno'],213 path.join(tmp_dir, 'work'), click, adb)
199 test_dir)214 else:
200215 manifest = adb.get_click_manifest(user)
216 if click:
217 print('Only setting up for %s' % click)
218 manifest = [entry for entry in manifest if click == entry['name']]
219 print('Only keeping entries with x-source')
220 manifest = [entry for entry in manifest if 'x-source' in entry]
221 for entry in manifest:
222 get_bzr_tests(entry['x-source']['vcs-bzr'],
223 entry['x-source']['vcs-bzr-revno'],
224 test_dir)
225
226def install_click_from_branch(branch, branch_path, click, adb):
227 #grab clickbuild tool
228 print("Calling click build with %s, source %s" % (branch, branch_path))
229 click_arg = "python3 click-build.py --bzr-source %s --source-directory %s" \
230 % (branch, branch_path)
231 os.system(click_arg)
232
233 click_arg = "ls | grep %s*.click" % click
234 click_name = check_output(click_arg, shell=True).strip()
235 click_path = path.join(branch_path, 'click_name')
236 shutil.move(click_name, click_path)
237 print("Pushing %s to device" % click_name)
238 adb.push(click_path, '/tmp')
239
240 print("Installing %s on device" % click_name)
241 installcmd="sudo -u phablet pkcon install-local /tmp/%s" % click_name
242 adb.shell(installcmd)
201243
202def main():244def main():
203 global lp245 global lp
204 args = parse_arguments()246 args = parse_arguments()
205 adb = UbuntuDevice(args.serial)247 adb = UbuntuDevice(args.serial)
206 adb.start()248 adb.start()
249 test_dir = tempfile.mkdtemp()
207 arch = adb.shell('dpkg --print-architecture').strip()250 arch = adb.shell('dpkg --print-architecture').strip()
208 series = adb.shell('lsb_release -c -s').strip()251 series = adb.shell('lsb_release -c -s').strip()
209 lp = LP(series, arch)252 lp = LP(series, arch)
210 test_dir = tempfile.mkdtemp()
211 atexit.register(cleanup, test_dir)253 atexit.register(cleanup, test_dir)
212 fetch_test_base(adb, test_dir)254 if args.bootstrap:
213 fetch_click_tests(adb, test_dir, args.user, args.click)255 fetch_test_base(adb, test_dir)
256 fetch_click_tests(adb, test_dir, args.user, args.click, args.branch)
214 destination = path.join('/home', args.user, 'autopilot')257 destination = path.join('/home', args.user, 'autopilot')
215 if args.wipe:258 if args.wipe:
216 print('Clearing previous test setup in %s' % destination)259 print('Clearing previous test setup in %s' % destination)
217260
=== added file 'po2mo.sh'
--- po2mo.sh 1970-01-01 00:00:00 +0000
+++ po2mo.sh 2014-01-14 22:09:32 +0000
@@ -0,0 +1,24 @@
1#!/bin/sh
2
3target=$1
4appname=$2
5path=$3
6
7processed=0
8for pofile in $(find $3 -name "*.po"); do
9 processed=1
10 pofilename="${pofile##*/}"
11 #echo "Processing $pofilename with $langcode"
12 langcode="${pofilename%.*}"
13 localedir="$target/locale/$langcode/LC_MESSAGES"
14 #echo "Making $localedir"
15 mkdir -p $localedir
16 mofile="$localedir/$appname.mo"
17 msgfmt -o "$mofile" "$pofile"
18done
19
20if [ processed ]; then
21 echo 'Found and processed po translations to locale folder'
22else
23 echo 'No translations found to process'
24fi

Subscribers

People subscribed via source and target branches