Merge ~litios/ubuntu-cve-tracker:aliases-v2 into ubuntu-cve-tracker:master
- Git
- lp:~litios/ubuntu-cve-tracker
- aliases-v2
- Merge into master
Status: | Merged |
---|---|
Merged at revision: | 6429624c17b76980ce95094140c2527fda8b1def |
Proposed branch: | ~litios/ubuntu-cve-tracker:aliases-v2 |
Merge into: | ubuntu-cve-tracker:master |
Diff against target: |
264 lines (+116/-39) 5 files modified
scripts/active_edit (+12/-4) scripts/check-cves (+18/-1) scripts/check-syntax (+24/-24) scripts/check-syntax-fixup (+29/-4) scripts/source_map.py (+33/-6) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Alex Murray | Approve | ||
Review via email: mp+437199@code.launchpad.net |
Commit message
Description of the change
This MR is to use aliases the other way around as it is right now:
* supported.txt will contain the name of the package in the subproject
* aliases.yaml will contain the aliases for the package name in Ubuntu
Example:
$ cat subprojects/
cryptography
$ cat subprojects/
---
cryptography: [python-
When doing triage, we want to make the process as easy as possible; therefore, the tool will expect that the user always provides the Ubuntu name of the package. It will warn the user if using the subproject name instead of the Ubuntu one (if it exists), as this will break any other aliases in other subprojects.
In case the package doesn't exist in Ubuntu, the tool will warn the user that it exists in a subproject and the person doing triage can provide the subproject name(s) so the CVE gets created.
Example output with check-cves and check-syntax: https:/
David Fernandez Gonzalez (litios) wrote : | # |
Alex Murray (alexmurray) wrote : | # |
A few questions inline.
David Fernandez Gonzalez (litios) wrote : | # |
Thanks for the input Alex!
I was relying too much on the user to do the manual work of figuring out the right alias when triaging.
I added some new commits to make it more transparent.
About the multi-versioned packages Steve mentioned, the last commit should also fix the issue.
David Fernandez Gonzalez (litios) wrote (last edit ): | # |
I added a final commit to deal with aliases in SOSS.
SOSS package names have the format PACKAGE|VERSION, and that's what the contents of supported.txt are.
Right now, the format of aliases.yaml would be something like this to match it:
"cryptography|
"cryptography|
...
but really, whatever is after the | doesn't seem useful for aliases, as the package is always the same. So the proposed format from @ccdm94 was:
"cryptography": ["python-
and that should cover all of the "cryptography|
Alex Murray (alexmurray) wrote : | # |
LGTM - but one minor query about whether we even need check-syntax-fixup anymore and hence could these changes to it be removed and the file deleted instead?
Alex Murray (alexmurray) wrote : | # |
Actually as pointed out by mdeslaur, check-syntax-fixup is still useful for a bunch of use-cases so please ignore those comments.
David Fernandez Gonzalez (litios) wrote : | # |
Thanks Alex!
Preview Diff
1 | diff --git a/scripts/active_edit b/scripts/active_edit | |||
2 | index 9d0df27..ec36cfe 100755 | |||
3 | --- a/scripts/active_edit | |||
4 | +++ b/scripts/active_edit | |||
5 | @@ -43,6 +43,8 @@ for release in cve_lib.all_releases: | |||
6 | 43 | source_releases.append(release) | 43 | source_releases.append(release) |
7 | 44 | source = source_map.load(releases=source_releases, skip_eol_releases=False) | 44 | source = source_map.load(releases=source_releases, skip_eol_releases=False) |
8 | 45 | 45 | ||
9 | 46 | added_rel_pkg = [] | ||
10 | 47 | |||
11 | 46 | def pkg_in_rel(pkg,rel): | 48 | def pkg_in_rel(pkg,rel): |
12 | 47 | if rel in ['upstream']: | 49 | if rel in ['upstream']: |
13 | 48 | return True | 50 | return True |
14 | @@ -88,10 +90,12 @@ def _add_pkg(p, fp, fixed, parent, embargoed): | |||
15 | 88 | if len(fixed[p]) > 2: | 90 | if len(fixed[p]) > 2: |
16 | 89 | fixed_in_release_version = fixed[p][2] | 91 | fixed_in_release_version = fixed[p][2] |
17 | 90 | for rel in ['upstream'] + list(source.keys()): | 92 | for rel in ['upstream'] + list(source.keys()): |
18 | 93 | aliases = source_map.get_aliases_of_ubuntu_package(source, p, rel) | ||
19 | 94 | |||
20 | 91 | # determine default state but override this if pkg_db has a | 95 | # determine default state but override this if pkg_db has a |
21 | 92 | # better one | 96 | # better one |
22 | 93 | state = "needs-triage" | 97 | state = "needs-triage" |
24 | 94 | if not pkg_in_rel(p, rel): | 98 | if not pkg_in_rel(p, rel) and not aliases: |
25 | 95 | # package doesn't exist in this release - see if it wants a | 99 | # package doesn't exist in this release - see if it wants a |
26 | 96 | # DNE entry | 100 | # DNE entry |
27 | 97 | if release_wants_dne(rel): | 101 | if release_wants_dne(rel): |
28 | @@ -128,9 +132,13 @@ def _add_pkg(p, fp, fixed, parent, embargoed): | |||
29 | 128 | print('%s_%s: %s' % (rel, p, state), file=fp) | 132 | print('%s_%s: %s' % (rel, p, state), file=fp) |
30 | 129 | # TODO: figure out how to store subproject details for embargoed CVEs | 133 | # TODO: figure out how to store subproject details for embargoed CVEs |
31 | 130 | elif not embargoed: | 134 | elif not embargoed: |
35 | 131 | # add this to subprojects for rel | 135 | pkgs_to_add = aliases if aliases else [p] |
36 | 132 | with open(os.path.join(cve_lib.get_external_subproject_cve_dir(rel), cve), "a") as f: | 136 | for pkg in pkgs_to_add: |
37 | 133 | print('%s_%s: %s' % (rel, p, state), file=f) | 137 | rel_pkgname = rel + '/' + pkg |
38 | 138 | if not rel_pkgname in added_rel_pkg: | ||
39 | 139 | with open(os.path.join(cve_lib.get_external_subproject_cve_dir(rel), cve), "a") as f: | ||
40 | 140 | print('%s_%s: %s' % (rel, pkg, state), file=f) | ||
41 | 141 | added_rel_pkg.append(rel_pkgname) | ||
42 | 134 | 142 | ||
43 | 135 | def add_pkg(p, fp, fixed, parent, embargoed): | 143 | def add_pkg(p, fp, fixed, parent, embargoed): |
44 | 136 | # skip packages which don't actually exist in any release | 144 | # skip packages which don't actually exist in any release |
45 | diff --git a/scripts/check-cves b/scripts/check-cves | |||
46 | index 6eb9f58..2bc15d2 100755 | |||
47 | --- a/scripts/check-cves | |||
48 | +++ b/scripts/check-cves | |||
49 | @@ -884,9 +884,26 @@ class CVEHandler(xml.sax.handler.ContentHandler): | |||
50 | 884 | external_subprojects = {} | 884 | external_subprojects = {} |
51 | 885 | for subproject in cve_lib.external_releases: | 885 | for subproject in cve_lib.external_releases: |
52 | 886 | if subproject in source: | 886 | if subproject in source: |
53 | 887 | aliases = source_map.get_all_aliases(source, subproject) | ||
54 | 887 | for hint in software_hints_from_cve_description: | 888 | for hint in software_hints_from_cve_description: |
56 | 888 | if hint in source[subproject] and hint not in common_words: | 889 | if hint in common_words: |
57 | 890 | continue | ||
58 | 891 | |||
59 | 892 | if hint in source[subproject]: | ||
60 | 893 | if 'aliases' in source[subproject][hint]: | ||
61 | 894 | external_subprojects.setdefault(subproject, set()).add(','.join(source[subproject][hint]['aliases'])) | ||
62 | 895 | else: | ||
63 | 896 | external_subprojects.setdefault(subproject, set()).add(hint) | ||
64 | 897 | elif hint in aliases: | ||
65 | 889 | external_subprojects.setdefault(subproject, set()).add(hint) | 898 | external_subprojects.setdefault(subproject, set()).add(hint) |
66 | 899 | else: | ||
67 | 900 | pkgs = source_map.get_packages_from_generic_name(source, hint, subproject) | ||
68 | 901 | for pkg in pkgs: | ||
69 | 902 | if 'aliases' in source[subproject][pkg]: | ||
70 | 903 | external_subprojects.setdefault(subproject, set()).add(','.join(source[subproject][pkg]['aliases'])) | ||
71 | 904 | else: | ||
72 | 905 | external_subprojects.setdefault(subproject, set()).add(pkg) | ||
73 | 906 | |||
74 | 890 | return external_subprojects | 907 | return external_subprojects |
75 | 891 | 908 | ||
76 | 892 | def display_cve(self, cve, file=sys.stdout, line_prefix=None, wrap_desc=False): | 909 | def display_cve(self, cve, file=sys.stdout, line_prefix=None, wrap_desc=False): |
77 | diff --git a/scripts/check-syntax b/scripts/check-syntax | |||
78 | index def63e3..4fb0cf8 100755 | |||
79 | --- a/scripts/check-syntax | |||
80 | +++ b/scripts/check-syntax | |||
81 | @@ -429,35 +429,35 @@ for cve in args: | |||
82 | 429 | 429 | ||
83 | 430 | # Check aliases | 430 | # Check aliases |
84 | 431 | for rel in all_required_releases: | 431 | for rel in all_required_releases: |
104 | 432 | if rel in source and pkg in source[rel] and \ | 432 | if rel in source: |
105 | 433 | 'aliases' in source[rel][pkg]: | 433 | aliases = source_map.get_aliases_of_ubuntu_package(source, pkg, rel) |
106 | 434 | aliases_releases.add(rel) | 434 | if aliases: |
107 | 435 | 435 | if rel in listed_releases and pkg \ | |
108 | 436 | # This package-release uses aliases, | 436 | not in aliases: |
90 | 437 | # it shouldn't be listed | ||
91 | 438 | if rel in listed_releases and pkg \ | ||
92 | 439 | not in source[rel][pkg]['aliases']: | ||
93 | 440 | filename = srcmap["pkgs"][pkg][nearby_rel][0] | ||
94 | 441 | linenum = srcmap["pkgs"][pkg][nearby_rel][1] | ||
95 | 442 | print( | ||
96 | 443 | "%s: %d: package '%s' not in '%s'" | ||
97 | 444 | % (filename, linenum, pkg, rel), | ||
98 | 445 | file=sys.stderr, | ||
99 | 446 | ) | ||
100 | 447 | |||
101 | 448 | failed = 0 | ||
102 | 449 | for alias in source[rel][pkg]['aliases']: | ||
103 | 450 | if alias not in data["pkgs"].keys(): | ||
109 | 451 | filename = srcmap["pkgs"][pkg][nearby_rel][0] | 437 | filename = srcmap["pkgs"][pkg][nearby_rel][0] |
110 | 452 | linenum = srcmap["pkgs"][pkg][nearby_rel][1] | 438 | linenum = srcmap["pkgs"][pkg][nearby_rel][1] |
111 | 453 | print( | 439 | print( |
115 | 454 | "%s: %d: %s missing release '%s'" | 440 | "%s: %d: package '%s' not in '%s'" |
116 | 455 | # put the error on a line near where this entry should go | 441 | % (filename, linenum, pkg, rel), |
114 | 456 | % (filename, linenum, alias, rel), | ||
117 | 457 | file=sys.stderr, | 442 | file=sys.stderr, |
118 | 458 | ) | 443 | ) |
121 | 459 | cve_okay = False | 444 | |
122 | 460 | failed += 1 | 445 | aliases_releases.add(rel) |
123 | 446 | |||
124 | 447 | # This package-release uses aliases, | ||
125 | 448 | # it shouldn't be listed | ||
126 | 449 | |||
127 | 450 | for alias in aliases: | ||
128 | 451 | if alias not in data["pkgs"].keys(): | ||
129 | 452 | filename = srcmap["pkgs"][pkg][nearby_rel][0] | ||
130 | 453 | linenum = srcmap["pkgs"][pkg][nearby_rel][1] | ||
131 | 454 | print( | ||
132 | 455 | "%s: %d: %s missing release '%s'" | ||
133 | 456 | # put the error on a line near where this entry should go | ||
134 | 457 | % (filename, linenum, alias, rel), | ||
135 | 458 | file=sys.stderr, | ||
136 | 459 | ) | ||
137 | 460 | cve_okay = False | ||
138 | 461 | 461 | ||
139 | 462 | missing_releases = all_required_releases - listed_releases - aliases_releases | 462 | missing_releases = all_required_releases - listed_releases - aliases_releases |
140 | 463 | nearby_rel = list(listed_releases - missing_releases)[0] | 463 | nearby_rel = list(listed_releases - missing_releases)[0] |
141 | diff --git a/scripts/check-syntax-fixup b/scripts/check-syntax-fixup | |||
142 | index c7c77c2..697368c 100755 | |||
143 | --- a/scripts/check-syntax-fixup | |||
144 | +++ b/scripts/check-syntax-fixup | |||
145 | @@ -59,11 +59,31 @@ def delete_from_file(filename: str, linenum: int, dryrun=False, verbose=False): | |||
146 | 59 | else: | 59 | else: |
147 | 60 | print("%s: %d: would delete this line" % (os.path.relpath(filename), linenum)) | 60 | print("%s: %d: would delete this line" % (os.path.relpath(filename), linenum)) |
148 | 61 | 61 | ||
149 | 62 | |||
150 | 63 | def identify_subproject_line_number(filename: str, pkg: str, rel: str): | ||
151 | 64 | # file may not already exist | ||
152 | 65 | contents = [] | ||
153 | 66 | try: | ||
154 | 67 | with open(filename, "r") as f: | ||
155 | 68 | contents = f.readlines() | ||
156 | 69 | except FileNotFoundError: | ||
157 | 70 | pass | ||
158 | 71 | |||
159 | 72 | for linenum,line in enumerate(contents): | ||
160 | 73 | line_rel, line_pkg = line.split(":")[0].split('_') | ||
161 | 74 | if pkg == line_pkg and rel == line_rel: | ||
162 | 75 | return linenum + 1 | ||
163 | 76 | |||
164 | 77 | return -1 | ||
165 | 78 | |||
166 | 62 | def get_pkg_rel_from_msg(msg): | 79 | def get_pkg_rel_from_msg(msg): |
167 | 63 | parts = msg.split(" ") | 80 | parts = msg.split(" ") |
168 | 64 | if 'DOES exist' in msg or 'not in' in msg: | 81 | if 'DOES exist' in msg or 'not in' in msg: |
169 | 65 | pkg = parts[1].replace("'", "") | 82 | pkg = parts[1].replace("'", "") |
170 | 66 | rel = parts[-1].replace("'", "") | 83 | rel = parts[-1].replace("'", "") |
171 | 84 | elif 'unknown package' in msg: | ||
172 | 85 | pkg = parts[2].replace("'", "") | ||
173 | 86 | rel = parts[-1].replace("'", "") | ||
174 | 67 | else: | 87 | else: |
175 | 68 | pkg = parts[0] | 88 | pkg = parts[0] |
176 | 69 | rel = parts[-1].replace("'", "") | 89 | rel = parts[-1].replace("'", "") |
177 | @@ -153,8 +173,11 @@ for line in args.infile: | |||
178 | 153 | cve = os.path.join( | 173 | cve = os.path.join( |
179 | 154 | cve_lib.get_external_subproject_cve_dir(rel), os.path.basename(cve) | 174 | cve_lib.get_external_subproject_cve_dir(rel), os.path.basename(cve) |
180 | 155 | ) | 175 | ) |
183 | 156 | # linenum is only relevant to the original cve file | 176 | linenum = identify_subproject_line_number(cve, pkg, rel) |
184 | 157 | linenum = 1 | 177 | if linenum == -1: |
185 | 178 | if 'DOES exist' in msg: | ||
186 | 179 | continue | ||
187 | 180 | linenum = 1 # We are inserting | ||
188 | 158 | 181 | ||
189 | 159 | # Remove the 'DNE' line before adding the new one | 182 | # Remove the 'DNE' line before adding the new one |
190 | 160 | if 'DOES exist' in msg: | 183 | if 'DOES exist' in msg: |
191 | @@ -171,8 +194,10 @@ for line in args.infile: | |||
192 | 171 | cve = os.path.join( | 194 | cve = os.path.join( |
193 | 172 | cve_lib.get_external_subproject_cve_dir(rel), os.path.basename(cve) | 195 | cve_lib.get_external_subproject_cve_dir(rel), os.path.basename(cve) |
194 | 173 | ) | 196 | ) |
197 | 174 | # linenum is only relevant to the original cve file | 197 | linenum = identify_subproject_line_number(cve, pkg, rel) |
198 | 175 | linenum = 1 | 198 | if linenum == -1: |
199 | 199 | print(line, file=sys.stderr) | ||
200 | 200 | continue | ||
201 | 176 | 201 | ||
202 | 177 | # delete this line since | 202 | # delete this line since |
203 | 178 | delete_from_file(cve, linenum, args.dry_run, args.verbose) | 203 | delete_from_file(cve, linenum, args.dry_run, args.verbose) |
204 | diff --git a/scripts/source_map.py b/scripts/source_map.py | |||
205 | index 311c1f6..616c5ec 100755 | |||
206 | --- a/scripts/source_map.py | |||
207 | +++ b/scripts/source_map.py | |||
208 | @@ -423,6 +423,29 @@ def get_built_using_header(): | |||
209 | 423 | header += "\n" + "-" * 78 | 423 | header += "\n" + "-" * 78 |
210 | 424 | return header | 424 | return header |
211 | 425 | 425 | ||
212 | 426 | def get_all_aliases(sources, rel): | ||
213 | 427 | aliases = [] | ||
214 | 428 | for pkg in sources[rel]: | ||
215 | 429 | if 'aliases' in sources[rel][pkg]: | ||
216 | 430 | for alias in sources[rel][pkg]['aliases']: | ||
217 | 431 | aliases.append(alias) | ||
218 | 432 | return aliases | ||
219 | 433 | |||
220 | 434 | def get_aliases_of_ubuntu_package(sources, pkg_name, rel_name): | ||
221 | 435 | aliases = [] | ||
222 | 436 | for rel in sources: | ||
223 | 437 | for pkg in sources[rel]: | ||
224 | 438 | if rel == rel_name and 'aliases' in sources[rel][pkg]: | ||
225 | 439 | if pkg_name in sources[rel][pkg]['aliases']: | ||
226 | 440 | aliases.append(pkg) | ||
227 | 441 | return aliases | ||
228 | 442 | |||
229 | 443 | def get_packages_from_generic_name(sources, generic_name, rel): | ||
230 | 444 | pkgs = [] | ||
231 | 445 | for pkg in sources[rel]: | ||
232 | 446 | if 'generic_name' in sources[rel][pkg] and sources[rel][pkg]['generic_name'] == generic_name: | ||
233 | 447 | pkgs.append(pkg) | ||
234 | 448 | return pkgs | ||
235 | 426 | 449 | ||
236 | 427 | def load_subprojects_lists(releases=None): | 450 | def load_subprojects_lists(releases=None): |
237 | 428 | map = dict() | 451 | map = dict() |
238 | @@ -471,17 +494,21 @@ def load_subprojects_lists(releases=None): | |||
239 | 471 | map[rel][pkg]['pocket'] = '' | 494 | map[rel][pkg]['pocket'] = '' |
240 | 472 | map[rel][pkg]['section'] = 'main' | 495 | map[rel][pkg]['section'] = 'main' |
241 | 473 | 496 | ||
242 | 497 | if '|' in pkg: | ||
243 | 498 | main_package_name = pkg.split('|')[0] | ||
244 | 499 | map[rel][pkg]['generic_name'] = main_package_name | ||
245 | 500 | |||
246 | 474 | if 'aliases' in details: | 501 | if 'aliases' in details: |
247 | 475 | with open(details['aliases'], 'r') as file: | 502 | with open(details['aliases'], 'r') as file: |
248 | 476 | aliases = yaml.safe_load(file) | 503 | aliases = yaml.safe_load(file) |
249 | 477 | 504 | ||
250 | 478 | for pkg in aliases: | 505 | for pkg in aliases: |
257 | 479 | if pkg in map[rel]: | 506 | for src_pkg in map[rel]: |
258 | 480 | map[rel][pkg]['aliases'] = aliases[pkg] | 507 | if pkg == src_pkg or \ |
259 | 481 | #else: | 508 | ('generic_name' in map[rel][src_pkg] and pkg == map[rel][src_pkg]['generic_name']): |
260 | 482 | # print("WARN: pkg %s found in aliases but not in supported list for %s. Skipping" % (pkg, rel)) | 509 | map[rel][src_pkg]['aliases'] = aliases[pkg] |
261 | 483 | else: | 510 | #else: |
262 | 484 | pass | 511 | # print("WARN: pkg %s found in aliases but not in supported list for %s. Skipping" % (pkg, rel)) |
263 | 485 | 512 | ||
264 | 486 | return map | 513 | return map |
265 | 487 | 514 |
It seems like the preview diff is broken, all the changes to CVEs are already merged in master.