Ubuntu Reports

Merge lp:~corey.bryant/ubuntu-reports/upstream into lp:ubuntu-reports

  1. upstream
  2. Merge into trunk
Proposed by Corey Bryant
Status: Merged
Merged at revision: 137
Proposed branch: lp:~corey.bryant/ubuntu-reports/upstream
Merge into: lp:ubuntu-reports
Diff against target: 342 lines (+211/-10)
4 files modified
server/cloud-archive/version-tracker/README.txt (+9/-0)
server/cloud-archive/version-tracker/ca-versions.py (+21/-7)
server/cloud-archive/version-tracker/common.py (+4/-0)
server/cloud-archive/version-tracker/gather-versions.py (+177/-3)
To merge this branch: bzr merge lp:~corey.bryant/ubuntu-reports/upstream
Reviewer Review Type Date Requested Status
Ubuntu Reports Dev Team Pending
Review via email: mp+302940@code.launchpad.net

Description of the change

This tool provides an easier way to track upstream releases vs watching mailing lists, etc. It's similar to the current cloud archive reports.

We may want to keep these reports behind the firewall. Currently I'm publishing reports here:
https://private-fileshare.canonical.com/~coreycb/ca_upstream_versions_newton.html
https://private-fileshare.canonical.com/~coreycb/ca_upstream_versions_mitaka.html
https://private-fileshare.canonical.com/~coreycb/ca_upstream_versions_liberty.html
https://private-fileshare.canonical.com/~coreycb/ca_upstream_versions_kilo.html

To post a comment you must log in.
lp:~corey.bryant/ubuntu-reports/upstream updated
134. By Brian Murray

switch incoming release tags

135. By Brian Murray

be smarter about which duplicate bugs we check who the reporter is, handle an inaccessible bug

136. By Corey Bryant

Add cloud-archive upstream version-tracker

Add support for tracking the latest upstream versions against current
package versions. The versions are normalized to enable version
comparison and the package list is trimmed to include OpenStack
deliverables only.

For more details see: server/cloud-archive/version-tracker/README.txt

137. By Corey Bryant

Add ocata for UCA reports

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'server/cloud-archive/version-tracker/README.txt'
2--- server/cloud-archive/version-tracker/README.txt 2013-12-03 20:52:26 +0000
3+++ server/cloud-archive/version-tracker/README.txt 2016-10-19 14:41:11 +0000
4@@ -9,6 +9,15 @@
5 ./gather-versions.py "--output_dir=$datadir" "$ca_item"
6 ./ca-versions.py "--json_dir=$datadir" --os_release="$ca_item"
7
8+Below is an example that includes upstream version details for Ocata, where the
9+versions are normalized and packages trimmed down to OpenStack deliverables only:
10+ datadir="$PWD/data";
11+ ca_item="ocata"
12+
13+ mkdir "$datadir"
14+ ./gather-versions.py "--output_dir=$datadir" "$ca_item" --upstream
15+ ./ca-versions.py "--json_dir=$datadir" --os_release="$ca_item" --upstream
16+
17 After doing so, output is in 'ca_versions_${ca_item}.html'
18
19 # output is in ca_versions_${ca_item}.html
20
21=== modified file 'server/cloud-archive/version-tracker/ca-versions.py'
22--- server/cloud-archive/version-tracker/ca-versions.py 2014-07-22 13:15:44 +0000
23+++ server/cloud-archive/version-tracker/ca-versions.py 2016-10-19 14:41:11 +0000
24@@ -59,10 +59,13 @@
25 return rc == 0
26
27
28-def load_sets(json_dir, release, sets=None):
29+def load_sets(json_dir, release, sets=None, include_upstream=False):
30 # this list determines which pockets to display, and in what order.
31 if sets is None:
32- sets = ['ubuntu', 'staging', 'proposed', 'updates']
33+ if include_upstream:
34+ sets = ['ubuntu', 'staging', 'proposed', 'updates', 'upstream']
35+ else:
36+ sets = ['ubuntu', 'staging', 'proposed', 'updates']
37 i = 1
38 package_sets = {}
39 for s in sets:
40@@ -75,8 +78,12 @@
41 i += 1
42 return package_sets
43
44-def render_table(package_sets, release):
45- table = '<html><body><center>Ubuntu Cloud Archive Version Tracker: '
46+def render_table(package_sets, opts):
47+ release = opts.os_release
48+ if opts.include_upstream:
49+ table = '<html><body><center>Ubuntu Cloud Archive Upstream Version Tracker: '
50+ else:
51+ table = '<html><body><center>Ubuntu Cloud Archive Version Tracker: '
52 if release == 'cloud-tools':
53 table += '<b>Cloud Tools</b><br>'
54 vers_suffix = '~ctools'
55@@ -191,6 +198,9 @@
56 action='store_true', default=False,
57 help='Ensure Ubuntu pockets do not contain newer '
58 'packages than what is at least in CA staging.')
59+parser.add_option('-u', '--upstream', dest='include_upstream',
60+ action='store_true', default=False,
61+ help='Include upstream version column in output.')
62 (opts, args) = parser.parse_args()
63
64 if not os.path.exists(opts.json_dir):
65@@ -203,7 +213,8 @@
66 if f['name'] == "staging":
67 f['header'] = "next"
68 else:
69- sets = load_sets(opts.json_dir, opts.os_release)
70+ sets = load_sets(opts.json_dir, opts.os_release,
71+ include_upstream=opts.include_upstream)
72
73 if opts.check_ubuntu:
74 # ensure there are no newer packages in the ubuntu release than in
75@@ -241,11 +252,14 @@
76
77
78 if not opts.out_file:
79- out_file = 'ca_versions_%s.html' % opts.os_release
80+ if opts.include_upstream:
81+ out_file = 'ca_upstream_versions_%s.html' % opts.os_release
82+ else:
83+ out_file = 'ca_versions_%s.html' % opts.os_release
84 else:
85 out_file = opts.out_file
86
87-html = render_table(sets, opts.os_release)
88+html = render_table(sets, opts)
89 out = open(out_file, 'w')
90 out.write(html)
91 out.close()
92
93=== modified file 'server/cloud-archive/version-tracker/common.py'
94--- server/cloud-archive/version-tracker/common.py 2016-05-31 08:45:05 +0000
95+++ server/cloud-archive/version-tracker/common.py 2016-10-19 14:41:11 +0000
96@@ -44,6 +44,10 @@
97 'ubuntu_stable': 'xenial',
98 'ubuntu_dev': 'yakkety',
99 },
100+ 'ocata': {
101+ 'ubuntu_stable': 'xenial',
102+ 'ubuntu_dev': 'zesty',
103+ },
104 'cloud-tools': {
105 'ubuntu_stable': CURRENT_UBUNTU_STABLE,
106 'ubuntu_dev': 'trusty',
107
108=== modified file 'server/cloud-archive/version-tracker/gather-versions.py'
109--- server/cloud-archive/version-tracker/gather-versions.py 2014-03-17 10:51:33 +0000
110+++ server/cloud-archive/version-tracker/gather-versions.py 2016-10-19 14:41:11 +0000
111@@ -14,19 +14,26 @@
112 # The package set to be tracked is determined by what currently exists in the
113 # staging PPA.
114 #
115+# Optionally include upstream version details, where versions are normalized
116+# and resulting packages trimmed down to OpenStack deliverables only.
117+#
118 # Version info is dumped to JSON files (one for each repo) to a specified
119 # directory ($HOME/.ca-versions) by default. These can be used to generate
120 # a HTML report using the ca-versions.py script.
121
122 import common
123+import git
124 import json
125 import os
126+import re
127 import sys
128 import logging
129 import httplib2
130 import optparse
131 import gzip
132+import shutil
133 import StringIO
134+import yaml
135
136 from launchpadlib.launchpad import Launchpad
137
138@@ -37,6 +44,9 @@
139 parser.add_option('-o', '--output_dir', action='store', dest='out_dir',
140 default=os.path.join(os.getenv('HOME'), '.ca-versions'),
141 help='Directory to dump JSON (Default: $HOME/.ca-versions)')
142+parser.add_option('-u', '--upstream', dest='include_upstream',
143+ action='store_true', default=False,
144+ help='Get upstream versions and normalize/trim results.')
145 (opts, args) = parser.parse_args()
146
147 if len(args) == 0:
148@@ -192,6 +202,122 @@
149 out[p] = vers
150 return out
151
152+def normalize_version(version):
153+ '''
154+ Convert a package version into its upstream version format, in order to
155+ enable version comparison.
156+ '''
157+ normalize_regular_expressions = {
158+ "^\d+:": "",
159+ "\-.*$": "",
160+ "\+dfsg.*$": "",
161+ "\~b1": ".0b1",
162+ "\~b2": ".0b2",
163+ "\~b3": ".0b3",
164+ "\~rc1": ".0rc1",
165+ "\~rc2": ".0rc2",
166+ "\~rc3": ".0rc3",
167+ "\~rc4": ".0rc4",
168+ "\~cloud.*$": "",
169+ "ubuntu.*$": "",
170+ }
171+ normalized_version = version
172+ if version:
173+ for pattern, replacement in normalize_regular_expressions.iteritems():
174+ normalized_version = re.sub(r"{}".format(pattern),
175+ replacement,
176+ normalized_version)
177+ return normalized_version
178+
179+# Map of ubuntu package names to their corresponding upstream project names.
180+# Most mappings are programatically determined in query_release_deliverables.
181+# This mapping is just for the tricky names.
182+PKG_TO_UPSTREAM_MAP = {
183+ 'openstack-trove': 'trove',
184+ 'oslo-sphinx': 'oslosphinx',
185+ 'python-django-openstack-auth': 'django_openstack_auth',
186+ 'python-keystoneauth1': 'keystoneauth',
187+}
188+
189+def _clone_repo(url, out_dir, sub_dir, branch=None):
190+ '''
191+ Clone a git repo.
192+ '''
193+ dest = os.path.join(out_dir, sub_dir)
194+ shutil.rmtree(dest, ignore_errors=True)
195+ if branch:
196+ git.Repo.clone_from(url, dest, branch=branch)
197+ else:
198+ git.Repo.clone_from(url, dest)
199+ return dest
200+
201+def query_release_deliverables(pkgs, release, out_dir):
202+ '''
203+ Loop through pkgs and get corresponding upstream versions from release
204+ repo's deliverables files.
205+ '''
206+ dest = _clone_repo('git://github.com/openstack/releases',
207+ out_dir, 'releases')
208+ release_dir = os.path.join(dest, 'deliverables', release)
209+ release_files = os.listdir(release_dir)
210+
211+ branch = 'stable/{}'.format(release)
212+ try:
213+ dest = _clone_repo('git://github.com/openstack/requirements',
214+ out_dir, 'requirements', branch)
215+ except git.exc.GitCommandError:
216+ # if stable branch not found, clone master branch
217+ branch = 'master'
218+ dest = _clone_repo('git://github.com/openstack/requirements',
219+ out_dir, 'requirements', branch)
220+ uc_file = os.path.join(dest, 'upper-constraints.txt')
221+
222+ upstream_versions = {}
223+ for pkg_name in pkgs:
224+ # generate upstream project name based on package name
225+ upstream_name = pkg_name
226+ if 'client' not in pkg_name:
227+ upstream_name = re.sub(r'python-', '', pkg_name)
228+ if pkg_name in PKG_TO_UPSTREAM_MAP.keys():
229+ upstream_name = PKG_TO_UPSTREAM_MAP[pkg_name]
230+
231+ release_file = "{}.yaml".format(upstream_name)
232+ if release_file not in release_files:
233+ continue
234+
235+ # Get latest version from project's release deliverable file.
236+ # Note: some projects exist in release deliverables but not in
237+ # upper-constraints.
238+ stream = file(os.path.join(release_dir, release_file), 'r')
239+ project_yaml = yaml.load(stream)
240+ upstream_vers = project_yaml['releases'][-1]['version']
241+
242+ # If version exists in upper-constraints, use that instead. Release
243+ # deliverables files can get updated before upper-constraints.
244+ stream = file(uc_file, 'r')
245+ for line in stream:
246+ lline = line.lower().rstrip()
247+ if upstream_name == re.sub(r'===.*$', '', lline):
248+ upstream_vers = re.sub(r'^.*===', '', lline)
249+
250+ upstream_versions[pkg_name] = upstream_vers
251+
252+ return upstream_versions
253+
254+def trim_packages_file(filename, packages):
255+ '''
256+ Trim the file to only include packages that are in the packages dict.
257+ '''
258+ f = open(filename, 'r')
259+
260+ pkgs = json.loads(f.read())
261+ for k in pkgs.keys():
262+ if k not in packages.keys():
263+ del pkgs[k]
264+
265+ f = open(filename, 'w')
266+ f.write(json.dumps(pkgs))
267+
268 stable_rel = "precise"
269
270 stable_rel = common.VERSION_MAP[os_release]['ubuntu_stable']
271@@ -212,7 +338,13 @@
272 staging_versions = query_ca_ppa(ppa=staging_ppa,
273 release=stable_rel)
274 f = open(os.path.join(opts.out_dir, '%s_staging_versions' % os_release), 'w+')
275-f.write(json.dumps(staging_versions))
276+if opts.include_upstream:
277+ staging_versions_normalized = {}
278+ for pkg_name, version in staging_versions.iteritems():
279+ staging_versions_normalized[pkg_name] = normalize_version(version[0])
280+ f.write(json.dumps(staging_versions_normalized))
281+else:
282+ f.write(json.dumps(staging_versions))
283 f.close()
284
285 # with the names of the packages published to the staging ppa, we can determine
286@@ -220,7 +352,13 @@
287 logging.info("Querying Ubuntu versions for all packages.")
288 ubuntu_versions = query_ubuntu(staging_versions.keys(), dev_rel)
289 f = open(os.path.join(opts.out_dir, "%s_ubuntu_versions" % os_release), 'w+')
290-f.write(json.dumps(ubuntu_versions))
291+if opts.include_upstream:
292+ ubuntu_versions_normalized = {}
293+ for pkg_name, version in ubuntu_versions.iteritems():
294+ ubuntu_versions_normalized[pkg_name] = normalize_version(version)
295+ f.write(json.dumps(ubuntu_versions_normalized))
296+else:
297+ f.write(json.dumps(ubuntu_versions))
298 f.close()
299
300 for p in pockets:
301@@ -228,4 +366,40 @@
302 versions = versions_from_packages(os_release, p, staging_versions.keys())
303 out = os.path.join(opts.out_dir, '%s_%s_versions' % (os_release, p))
304 f = open(out, 'w+')
305- f.write(json.dumps(versions))
306+ versions_normalized = {}
307+ if opts.include_upstream:
308+ for pkg_name, version in versions.iteritems():
309+ normalized_version = version
310+ if version:
311+ normalized_version = normalize_version(version)
312+ versions_normalized[pkg_name] = normalized_version
313+ f.write(json.dumps(versions_normalized))
314+ else:
315+ f.write(json.dumps(versions))
316+
317+if opts.include_upstream:
318+ # with the names of the packages published to the staging ppa, we can determine
319+ # the upstream versions we need to track.
320+ logging.info("Querying upstream versions for packages from release deliverables repo.")
321+ upstream_versions = query_release_deliverables(staging_versions,
322+ os_release, opts.out_dir)
323+
324+ # trim the results to only include packages that we've detected upstream
325+ # versions for
326+ for k in upstream_versions.keys():
327+ if upstream_versions[k] == None:
328+ del upstream_versions[k]
329+
330+ out = os.path.join(opts.out_dir, '%s_upstream_versions' % os_release)
331+ f = open(out, 'w')
332+ f.write(json.dumps(upstream_versions))
333+
334+ out = os.path.join(opts.out_dir, '%s_staging_versions' % os_release)
335+ trim_packages_file(out, upstream_versions)
336+
337+ out = os.path.join(opts.out_dir, '%s_ubuntu_versions' % os_release)
338+ trim_packages_file(out, upstream_versions)
339+
340+ for p in pockets:
341+ out = os.path.join(opts.out_dir, '%s_%s_versions' % (os_release, p))
342+ trim_packages_file(out, upstream_versions)

Subscribers

People subscribed via source and target branches