Merge ~litios/ubuntu-cve-tracker:ppas-tag-migration into ubuntu-cve-tracker:master
- Git
- lp:~litios/ubuntu-cve-tracker
- ppas-tag-migration
- Merge into master
Proposed by
David Fernandez Gonzalez
Status: | Merged |
---|---|
Merge reported by: | David Fernandez Gonzalez |
Merged at revision: | 8d5aa3898e261c0861d302c17b9d8220970388a8 |
Proposed branch: | ~litios/ubuntu-cve-tracker:ppas-tag-migration |
Merge into: | ubuntu-cve-tracker:master |
Diff against target: |
435 lines (+114/-73) 3 files modified
scripts/cve_lib.py (+78/-45) scripts/generate_pkg_cache.py (+35/-27) scripts/test_source_map.py (+1/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Eduardo Barretto | Approve | ||
Emilia Torino | Approve | ||
Review via email: mp+453042@code.launchpad.net |
Commit message
Description of the change
Due to https:/
To post a comment you must log in.
Revision history for this message
Eduardo Barretto (ebarretto) wrote : | # |
As we've spoken, the current changes break generate_
It would also be great to handle the pockets correctly in that same script, otherwise everything will show up as "Release" pocket.
Thanks for making these changes
review:
Needs Fixing
- be2fc23... by David Fernandez Gonzalez
-
Add pockets to subprojects
Signed-off-by: David Fernandez Gonzalez <email address hidden>
- 8d5aa38... by David Fernandez Gonzalez
-
cve_lib: remove /ubuntu from subproject ppa names
Signed-off-by: David Fernandez Gonzalez <email address hidden>
Revision history for this message
Eduardo Barretto (ebarretto) wrote : | # |
lgtm, thanks
CI failures are not related to this PR
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py | |||
2 | index fb9a744..7299ac0 100755 | |||
3 | --- a/scripts/cve_lib.py | |||
4 | +++ b/scripts/cve_lib.py | |||
5 | @@ -106,7 +106,7 @@ subprojects = { | |||
6 | 106 | "name": "Ubuntu 12.04 ESM", | 106 | "name": "Ubuntu 12.04 ESM", |
7 | 107 | "codename": "Precise Pangolin", | 107 | "codename": "Precise Pangolin", |
8 | 108 | "alias": "precise/esm", | 108 | "alias": "precise/esm", |
10 | 109 | "ppa": "ubuntu-esm/esm", | 109 | "ppas": [{ "ppa": "ubuntu-esm/esm", "pocket": "security"}], |
11 | 110 | "parent": "ubuntu/precise", | 110 | "parent": "ubuntu/precise", |
12 | 111 | "description": "Available with UA Infra or UA Desktop: https://ubuntu.com/advantage", | 111 | "description": "Available with UA Infra or UA Desktop: https://ubuntu.com/advantage", |
13 | 112 | "stamp": 1493521200, | 112 | "stamp": 1493521200, |
14 | @@ -118,7 +118,10 @@ subprojects = { | |||
15 | 118 | "name": "Ubuntu 14.04 ESM", | 118 | "name": "Ubuntu 14.04 ESM", |
16 | 119 | "codename": "Trusty Tahr", | 119 | "codename": "Trusty Tahr", |
17 | 120 | "alias": "trusty/esm", | 120 | "alias": "trusty/esm", |
19 | 121 | "ppa": "ubuntu-esm/esm-infra-security", | 121 | "ppas": [ |
20 | 122 | {"ppa": "ubuntu-esm/esm-infra-security", "pocket": "security"}, | ||
21 | 123 | {"ppa": "ubuntu-esm/esm-infra-updates", "pocket": "updates"} | ||
22 | 124 | ], | ||
23 | 122 | "parent": "ubuntu/trusty", | 125 | "parent": "ubuntu/trusty", |
24 | 123 | "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro", | 126 | "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro", |
25 | 124 | "stamp": 1556593200, | 127 | "stamp": 1556593200, |
26 | @@ -130,7 +133,10 @@ subprojects = { | |||
27 | 130 | "packages": ["esm-infra-xenial-supported.txt"], | 133 | "packages": ["esm-infra-xenial-supported.txt"], |
28 | 131 | "name": "Ubuntu 16.04 ESM", | 134 | "name": "Ubuntu 16.04 ESM", |
29 | 132 | "codename": "Xenial Xerus", | 135 | "codename": "Xenial Xerus", |
31 | 133 | "ppa": "ubuntu-esm/esm-infra-security", | 136 | "ppas": [ |
32 | 137 | {"ppa": "ubuntu-esm/esm-infra-security", "pocket": "security"}, | ||
33 | 138 | {"ppa": "ubuntu-esm/esm-infra-updates", "pocket": "updates"} | ||
34 | 139 | ], | ||
35 | 134 | "parent": "ubuntu/xenial", | 140 | "parent": "ubuntu/xenial", |
36 | 135 | "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro", | 141 | "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro", |
37 | 136 | "stamp": 1618963200, | 142 | "stamp": 1618963200, |
38 | @@ -142,7 +148,10 @@ subprojects = { | |||
39 | 142 | "packages": ["esm-infra-bionic-supported.txt"], | 148 | "packages": ["esm-infra-bionic-supported.txt"], |
40 | 143 | "name": "Ubuntu 18.04 ESM", | 149 | "name": "Ubuntu 18.04 ESM", |
41 | 144 | "codename": "Bionic Beaver", | 150 | "codename": "Bionic Beaver", |
43 | 145 | "ppa": "ubuntu-esm/esm-infra-security", | 151 | "ppas": [ |
44 | 152 | {"ppa": "ubuntu-esm/esm-infra-security", "pocket": "security"}, | ||
45 | 153 | {"ppa": "ubuntu-esm/esm-infra-updates", "pocket": "updates"} | ||
46 | 154 | ], | ||
47 | 146 | "parent": "ubuntu/bionic", | 155 | "parent": "ubuntu/bionic", |
48 | 147 | "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro", | 156 | "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro", |
49 | 148 | "stamp": 1685539024, | 157 | "stamp": 1685539024, |
50 | @@ -154,7 +163,10 @@ subprojects = { | |||
51 | 154 | "packages": ["esm-apps-xenial-supported.txt"], | 163 | "packages": ["esm-apps-xenial-supported.txt"], |
52 | 155 | "name": "Ubuntu 16.04 ESM", | 164 | "name": "Ubuntu 16.04 ESM", |
53 | 156 | "codename": "Xenial Xerus", | 165 | "codename": "Xenial Xerus", |
55 | 157 | "ppa": "ubuntu-esm/esm-apps-security", | 166 | "ppas": [ |
56 | 167 | {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, | ||
57 | 168 | {"ppa": "ubuntu-esm/esm-apps-updates", "pocket": "updates"} | ||
58 | 169 | ], | ||
59 | 158 | "parent": "esm-infra/xenial", | 170 | "parent": "esm-infra/xenial", |
60 | 159 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 171 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
61 | 160 | "stamp": 1618963200, | 172 | "stamp": 1618963200, |
62 | @@ -166,7 +178,10 @@ subprojects = { | |||
63 | 166 | "packages": ["esm-apps-bionic-supported.txt"], | 178 | "packages": ["esm-apps-bionic-supported.txt"], |
64 | 167 | "name": "Ubuntu 18.04 ESM", | 179 | "name": "Ubuntu 18.04 ESM", |
65 | 168 | "codename": "Bionic Beaver", | 180 | "codename": "Bionic Beaver", |
67 | 169 | "ppa": "ubuntu-esm/esm-apps-security", | 181 | "ppas": [ |
68 | 182 | {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, | ||
69 | 183 | {"ppa": "ubuntu-esm/esm-apps-updates", "pocket": "updates"} | ||
70 | 184 | ], | ||
71 | 170 | "parent": "esm-infra/bionic", | 185 | "parent": "esm-infra/bionic", |
72 | 171 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 186 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
73 | 172 | "stamp": 1524870000, | 187 | "stamp": 1524870000, |
74 | @@ -178,7 +193,10 @@ subprojects = { | |||
75 | 178 | "packages": ["esm-apps-focal-supported.txt"], | 193 | "packages": ["esm-apps-focal-supported.txt"], |
76 | 179 | "name": "Ubuntu 20.04 ESM", | 194 | "name": "Ubuntu 20.04 ESM", |
77 | 180 | "codename": "Focal Fossa", | 195 | "codename": "Focal Fossa", |
79 | 181 | "ppa": "ubuntu-esm/esm-apps-security", | 196 | "ppas": [ |
80 | 197 | {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, | ||
81 | 198 | {"ppa": "ubuntu-esm/esm-apps-updates", "pocket": "updates"} | ||
82 | 199 | ], | ||
83 | 182 | "parent": "ubuntu/focal", | 200 | "parent": "ubuntu/focal", |
84 | 183 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 201 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
85 | 184 | "stamp": 1587567600, | 202 | "stamp": 1587567600, |
86 | @@ -190,7 +208,10 @@ subprojects = { | |||
87 | 190 | "packages": ["esm-apps-jammy-supported.txt"], | 208 | "packages": ["esm-apps-jammy-supported.txt"], |
88 | 191 | "name": "Ubuntu 22.04 ESM", | 209 | "name": "Ubuntu 22.04 ESM", |
89 | 192 | "codename": "Jammy Jellyfish", | 210 | "codename": "Jammy Jellyfish", |
91 | 193 | "ppa": "ubuntu-esm/esm-apps-security", | 211 | "ppas": [ |
92 | 212 | {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, | ||
93 | 213 | {"ppa": "ubuntu-esm/esm-apps-updates", "pocket": "updates"} | ||
94 | 214 | ], | ||
95 | 194 | "parent": "ubuntu/jammy", | 215 | "parent": "ubuntu/jammy", |
96 | 195 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 216 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
97 | 196 | "stamp": 1650693600, | 217 | "stamp": 1650693600, |
98 | @@ -201,7 +222,7 @@ subprojects = { | |||
99 | 201 | "packages": ["fips-xenial-supported.txt"], | 222 | "packages": ["fips-xenial-supported.txt"], |
100 | 202 | "name": "Ubuntu 16.04 FIPS Certified", | 223 | "name": "Ubuntu 16.04 FIPS Certified", |
101 | 203 | "codename": "Xenial Xerus", | 224 | "codename": "Xenial Xerus", |
103 | 204 | "ppa": "ubuntu-advantage/fips", | 225 | "ppas": [{"ppa" : "ubuntu-advantage/fips", "pocket": "security"}], |
104 | 205 | "parent": "ubuntu/xenial", | 226 | "parent": "ubuntu/xenial", |
105 | 206 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 227 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
106 | 207 | }, | 228 | }, |
107 | @@ -211,7 +232,7 @@ subprojects = { | |||
108 | 211 | "packages": ["fips-bionic-supported.txt"], | 232 | "packages": ["fips-bionic-supported.txt"], |
109 | 212 | "name": "Ubuntu 18.04 FIPS Certified", | 233 | "name": "Ubuntu 18.04 FIPS Certified", |
110 | 213 | "codename": "Bionic Beaver", | 234 | "codename": "Bionic Beaver", |
112 | 214 | "ppa": "ubuntu-advantage/fips", | 235 | "ppas": [{"ppa" : "ubuntu-advantage/fips", "pocket": "security"}], |
113 | 215 | "parent": "ubuntu/bionic", | 236 | "parent": "ubuntu/bionic", |
114 | 216 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 237 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
115 | 217 | }, | 238 | }, |
116 | @@ -221,7 +242,7 @@ subprojects = { | |||
117 | 221 | "packages": ["fips-focal-supported.txt"], | 242 | "packages": ["fips-focal-supported.txt"], |
118 | 222 | "name": "Ubuntu 20.04 FIPS Certified", | 243 | "name": "Ubuntu 20.04 FIPS Certified", |
119 | 223 | "codename": "Focal Fossa", | 244 | "codename": "Focal Fossa", |
121 | 224 | "ppa": "ubuntu-advantage/fips", | 245 | "ppas": [{"ppa" : "ubuntu-advantage/fips", "pocket": "security"}], |
122 | 225 | "parent": "ubuntu/bionic", | 246 | "parent": "ubuntu/bionic", |
123 | 226 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 247 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
124 | 227 | }, | 248 | }, |
125 | @@ -231,7 +252,7 @@ subprojects = { | |||
126 | 231 | "packages": ["fips-updates-xenial-supported.txt"], | 252 | "packages": ["fips-updates-xenial-supported.txt"], |
127 | 232 | "name": "Ubuntu 16.04 FIPS Compliant", | 253 | "name": "Ubuntu 16.04 FIPS Compliant", |
128 | 233 | "codename": "Xenial Xerus", | 254 | "codename": "Xenial Xerus", |
130 | 234 | "ppa": "ubuntu-advantage/fips-updates", | 255 | "ppas": [{"ppa" : "ubuntu-advantage/fips-updates", "pocket": "updates"}], |
131 | 235 | "parent": "ubuntu/xenial", | 256 | "parent": "ubuntu/xenial", |
132 | 236 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 257 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
133 | 237 | }, | 258 | }, |
134 | @@ -241,7 +262,7 @@ subprojects = { | |||
135 | 241 | "packages": ["fips-updates-bionic-supported.txt"], | 262 | "packages": ["fips-updates-bionic-supported.txt"], |
136 | 242 | "name": "Ubuntu 18.04 FIPS Compliant", | 263 | "name": "Ubuntu 18.04 FIPS Compliant", |
137 | 243 | "codename": "Bionic Beaver", | 264 | "codename": "Bionic Beaver", |
139 | 244 | "ppa": "ubuntu-advantage/fips-updates", | 265 | "ppas": [{"ppa" : "ubuntu-advantage/fips-updates", "pocket": "updates"}], |
140 | 245 | "parent": "ubuntu/bionic", | 266 | "parent": "ubuntu/bionic", |
141 | 246 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 267 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
142 | 247 | }, | 268 | }, |
143 | @@ -251,7 +272,7 @@ subprojects = { | |||
144 | 251 | "packages": ["fips-updates-focal-supported.txt"], | 272 | "packages": ["fips-updates-focal-supported.txt"], |
145 | 252 | "name": "Ubuntu 20.04 FIPS Compliant", | 273 | "name": "Ubuntu 20.04 FIPS Compliant", |
146 | 253 | "codename": "Focal Fossa", | 274 | "codename": "Focal Fossa", |
148 | 254 | "ppa": "ubuntu-advantage/fips-updates", | 275 | "ppas": [{"ppa" : "ubuntu-advantage/fips-updates", "pocket": "updates"}], |
149 | 255 | "parent": "ubuntu/bionic", | 276 | "parent": "ubuntu/bionic", |
150 | 256 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 277 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
151 | 257 | }, | 278 | }, |
152 | @@ -262,7 +283,7 @@ subprojects = { | |||
153 | 262 | "name": "Ubuntu 16.04 ROS ESM", | 283 | "name": "Ubuntu 16.04 ROS ESM", |
154 | 263 | "codename": "Xenial Xerus", | 284 | "codename": "Xenial Xerus", |
155 | 264 | "alias": "ros-esm/xenial", | 285 | "alias": "ros-esm/xenial", |
157 | 265 | "ppa": "ubuntu-robotics-packagers/ros-security", | 286 | "ppas": [{"ppa": "ubuntu-robotics-packagers/ros-security", "pocket": "security"}], |
158 | 266 | "parent": "ubuntu/xenial", | 287 | "parent": "ubuntu/xenial", |
159 | 267 | "description": "Available with Ubuntu Advantage: https://ubuntu.com/advantage", | 288 | "description": "Available with Ubuntu Advantage: https://ubuntu.com/advantage", |
160 | 268 | }, | 289 | }, |
161 | @@ -273,7 +294,7 @@ subprojects = { | |||
162 | 273 | "name": "Ubuntu 18.04 ROS ESM", | 294 | "name": "Ubuntu 18.04 ROS ESM", |
163 | 274 | "codename": "Bionic Beaver", | 295 | "codename": "Bionic Beaver", |
164 | 275 | "alias": "ros-esm/bionic", | 296 | "alias": "ros-esm/bionic", |
166 | 276 | "ppa": "ubuntu-robotics-packagers/ros-security", | 297 | "ppas": [{"ppa": "ubuntu-robotics-packagers/ros-security", "pocket": "security"}], |
167 | 277 | "parent": "ubuntu/bionic", | 298 | "parent": "ubuntu/bionic", |
168 | 278 | "description": "Available with Ubuntu Advantage: https://ubuntu.com/advantage", | 299 | "description": "Available with Ubuntu Advantage: https://ubuntu.com/advantage", |
169 | 279 | }, | 300 | }, |
170 | @@ -755,17 +776,30 @@ def get_subproject_details_by_ppa_url_and_series(url, series): | |||
171 | 755 | prod, ser = product_series(rel) | 776 | prod, ser = product_series(rel) |
172 | 756 | if ser == series: | 777 | if ser == series: |
173 | 757 | try: | 778 | try: |
179 | 758 | if subprojects[rel]["ppa"] in url: | 779 | for ppa_pocket in subprojects[rel]["ppas"]: |
180 | 759 | product = prod | 780 | if ppa_pocket['ppa'] in url: |
181 | 760 | canon = product + "/" + series | 781 | product = prod |
182 | 761 | details = subprojects[rel] | 782 | canon = product + "/" + series |
183 | 762 | break | 783 | details = subprojects[rel] |
184 | 784 | break | ||
185 | 763 | except KeyError: | 785 | except KeyError: |
186 | 764 | pass | 786 | pass |
187 | 765 | if details is not None: | 787 | if details is not None: |
188 | 766 | break | 788 | break |
189 | 767 | return canon, product, series, details | 789 | return canon, product, series, details |
190 | 768 | 790 | ||
191 | 791 | def release_ppa_pocket(rel, ppa): | ||
192 | 792 | _, _, _, details = get_subproject_details(rel) | ||
193 | 793 | pocket = None | ||
194 | 794 | try: | ||
195 | 795 | for ppa_pocket in details["ppas"]: | ||
196 | 796 | if ppa_pocket["ppa"] == ppa: | ||
197 | 797 | pocket = ppa_pocket["pocket"] | ||
198 | 798 | break | ||
199 | 799 | except (KeyError, TypeError): | ||
200 | 800 | pass | ||
201 | 801 | return pocket | ||
202 | 802 | |||
203 | 769 | def release_name(rel): | 803 | def release_name(rel): |
204 | 770 | name = None | 804 | name = None |
205 | 771 | _, _, _, details = get_subproject_details(rel) | 805 | _, _, _, details = get_subproject_details(rel) |
206 | @@ -832,16 +866,16 @@ def release_version(rel): | |||
207 | 832 | if details: | 866 | if details: |
208 | 833 | version = details["version"] | 867 | version = details["version"] |
209 | 834 | return version | 868 | return version |
214 | 835 | 869 | ||
215 | 836 | def release_ppa(rel): | 870 | def release_ppas(rel): |
216 | 837 | """Return the ppa for a given subproject.""" | 871 | """Return the ppas for a given subproject.""" |
217 | 838 | ppa = None | 872 | ppas = None |
218 | 839 | _, _, _, details = get_subproject_details(rel) | 873 | _, _, _, details = get_subproject_details(rel) |
219 | 840 | try: | 874 | try: |
221 | 841 | ppa = details["ppa"] | 875 | ppas = details["ppas"] |
222 | 842 | except (KeyError, TypeError): | 876 | except (KeyError, TypeError): |
223 | 843 | pass | 877 | pass |
225 | 844 | return ppa | 878 | return ppas |
226 | 845 | 879 | ||
227 | 846 | def needs_oval(rel): | 880 | def needs_oval(rel): |
228 | 847 | """Return if OVAL should be generated for a given subproject""" | 881 | """Return if OVAL should be generated for a given subproject""" |
229 | @@ -931,7 +965,7 @@ def find_external_subproject_cves(cve): | |||
230 | 931 | # should follow the same as any other subproject | 965 | # should follow the same as any other subproject |
231 | 932 | # except for the extra 'product' and 'release' keys. | 966 | # except for the extra 'product' and 'release' keys. |
232 | 933 | MANDATORY_EXTERNAL_SUBPROJECT_KEYS = ['cve_triage', 'cve_patching', 'cve_notification', 'security_updates_notification', 'binary_copies_only', 'seg_support', 'owners'] | 967 | MANDATORY_EXTERNAL_SUBPROJECT_KEYS = ['cve_triage', 'cve_patching', 'cve_notification', 'security_updates_notification', 'binary_copies_only', 'seg_support', 'owners'] |
234 | 934 | MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS = ['ppa', 'oval', 'product', 'release', 'supported_packages'] | 968 | MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS = ['ppas', 'oval', 'product', 'release', 'supported_packages'] |
235 | 935 | OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS = ['parent', 'name', 'codename', 'description', 'aliases', 'archs'] | 969 | OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS = ['parent', 'name', 'codename', 'description', 'aliases', 'archs'] |
236 | 936 | 970 | ||
237 | 937 | def load_external_subprojects(strict=False): | 971 | def load_external_subprojects(strict=False): |
238 | @@ -976,8 +1010,8 @@ def load_external_subprojects(strict=False): | |||
239 | 976 | else: | 1010 | else: |
240 | 977 | support_metadata[key] = main_config[key] | 1011 | support_metadata[key] = main_config[key] |
241 | 978 | 1012 | ||
244 | 979 | for ppa in main_config['ppas']: | 1013 | for subproject in main_config['subprojects']: |
245 | 980 | config = main_config['ppas'][ppa] | 1014 | config = main_config['subprojects'][subproject] |
246 | 981 | if 'product' not in config or 'release' not in config: | 1015 | if 'product' not in config or 'release' not in config: |
247 | 982 | if strict: | 1016 | if strict: |
248 | 983 | raise ValueError(error_msg) | 1017 | raise ValueError(error_msg) |
249 | @@ -985,39 +1019,38 @@ def load_external_subprojects(strict=False): | |||
250 | 985 | print(error_msg, file=sys.stderr) | 1019 | print(error_msg, file=sys.stderr) |
251 | 986 | error_msg = '%s: missing "product" or "release".' % (subproject_path) | 1020 | error_msg = '%s: missing "product" or "release".' % (subproject_path) |
252 | 987 | 1021 | ||
256 | 988 | subproject_name = ppa | 1022 | external_releases.append(subproject) |
257 | 989 | external_releases.append(subproject_name) | 1023 | subprojects.setdefault(subproject, {"packages": [], |
255 | 990 | subprojects.setdefault(subproject_name, {"packages": [], | ||
258 | 991 | "eol": False}) | 1024 | "eol": False}) |
259 | 992 | # an external subproject can append to an internal one | 1025 | # an external subproject can append to an internal one |
261 | 993 | subprojects[subproject_name]["packages"].append(\ | 1026 | subprojects[subproject]["packages"].append(\ |
262 | 994 | os.path.join(subproject_path, config['supported_packages'])) | 1027 | os.path.join(subproject_path, config['supported_packages'])) |
263 | 995 | 1028 | ||
264 | 996 | # check if aliases for packages exist | 1029 | # check if aliases for packages exist |
265 | 997 | if 'aliases' in config: | 1030 | if 'aliases' in config: |
267 | 998 | subprojects[subproject_name].setdefault("aliases", \ | 1031 | subprojects[subproject].setdefault("aliases", \ |
268 | 999 | os.path.join(subproject_path, config['aliases'])) | 1032 | os.path.join(subproject_path, config['aliases'])) |
269 | 1000 | 1033 | ||
270 | 1001 | for key in MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS + OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS: | 1034 | for key in MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS + OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS: |
271 | 1002 | if key in config: | 1035 | if key in config: |
273 | 1003 | subprojects[subproject_name].setdefault(key, config[key]) | 1036 | subprojects[subproject].setdefault(key, config[key]) |
274 | 1004 | elif key in OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS: | 1037 | elif key in OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS: |
276 | 1005 | _, _, _, original_release_details = get_subproject_details(subprojects[subproject_name]['release']) | 1038 | _, _, _, original_release_details = get_subproject_details(subprojects[subproject]['release']) |
277 | 1006 | if original_release_details and key in original_release_details: | 1039 | if original_release_details and key in original_release_details: |
279 | 1007 | subprojects[subproject_name].setdefault(key, original_release_details[key]) | 1040 | subprojects[subproject].setdefault(key, original_release_details[key]) |
280 | 1008 | else: | 1041 | else: |
281 | 1009 | error_msg = '%s missing "%s" field.' % (subproject_path, key) | 1042 | error_msg = '%s missing "%s" field.' % (subproject_path, key) |
284 | 1010 | del subprojects[subproject_name] | 1043 | del subprojects[subproject] |
285 | 1011 | external_releases.remove(subproject_name) | 1044 | external_releases.remove(subproject) |
286 | 1012 | if strict: | 1045 | if strict: |
287 | 1013 | raise ValueError(error_msg) | 1046 | raise ValueError(error_msg) |
288 | 1014 | else: | 1047 | else: |
289 | 1015 | print(error_msg, file=sys.stderr) | 1048 | print(error_msg, file=sys.stderr) |
290 | 1016 | 1049 | ||
293 | 1017 | subprojects[subproject_name].setdefault("support_metadata", support_metadata) | 1050 | subprojects[subproject].setdefault("support_metadata", support_metadata) |
294 | 1018 | project = read_external_subproject_details(subproject_name) | 1051 | project = read_external_subproject_details(subproject) |
295 | 1019 | if project: | 1052 | if project: |
297 | 1020 | subprojects[subproject_name].setdefault("customer", project) | 1053 | subprojects[subproject].setdefault("customer", project) |
298 | 1021 | 1054 | ||
299 | 1022 | load_external_subprojects() | 1055 | load_external_subprojects() |
300 | 1023 | 1056 | ||
301 | @@ -1068,8 +1101,8 @@ def release_is_older_than(release_a, release_b): | |||
302 | 1068 | flavor_releases = [ | 1101 | flavor_releases = [ |
303 | 1069 | 'lucid', 'precise', 'trusty', 'utopic', 'vivid', 'wily', 'xenial', | 1102 | 'lucid', 'precise', 'trusty', 'utopic', 'vivid', 'wily', 'xenial', |
304 | 1070 | 'yakkety', 'zesty', 'artful', 'bionic', 'cosmic', 'disco', 'eoan', | 1103 | 'yakkety', 'zesty', 'artful', 'bionic', 'cosmic', 'disco', 'eoan', |
307 | 1071 | 'focal', 'groovy', 'hirsute', 'impish', 'jammy', 'kinetic', "lunar", | 1104 | 'focal', 'groovy', 'hirsute', 'impish', 'jammy', 'kinetic', 'lunar', |
308 | 1072 | 'mantic', 'noble', | 1105 | 'mantic','noble', |
309 | 1073 | ] | 1106 | ] |
310 | 1074 | 1107 | ||
311 | 1075 | all_releases = release_sort(all_releases) | 1108 | all_releases = release_sort(all_releases) |
312 | diff --git a/scripts/generate_pkg_cache.py b/scripts/generate_pkg_cache.py | |||
313 | index 27404f7..672e614 100644 | |||
314 | --- a/scripts/generate_pkg_cache.py | |||
315 | +++ b/scripts/generate_pkg_cache.py | |||
316 | @@ -11,7 +11,7 @@ | |||
317 | 11 | # for details. | 11 | # for details. |
318 | 12 | # | 12 | # |
319 | 13 | 13 | ||
321 | 14 | from cve_lib import (all_releases, devel_release, eol_releases, needs_oval, product_series, release_parent, release_ppa) | 14 | from cve_lib import (all_releases, devel_release, eol_releases, needs_oval, product_series, release_parent, release_ppas, release_ppa_pocket) |
322 | 15 | 15 | ||
323 | 16 | import argparse | 16 | import argparse |
324 | 17 | import datetime | 17 | import datetime |
325 | @@ -55,31 +55,11 @@ def write_to_cache(cache_dir, release, cache): | |||
326 | 55 | except Exception: | 55 | except Exception: |
327 | 56 | error(f"Could not write to JSON file: {filename}") | 56 | error(f"Could not write to JSON file: {filename}") |
328 | 57 | 57 | ||
351 | 58 | 58 | def get_binaries_rel(release, archive, real_threshold, series, cache, latest_date_created, ppa=None): | |
330 | 59 | def update_cache(release, cache, ppa=None, latest_date_created=None): | ||
331 | 60 | lp = lpl_common.connect(version='devel') | ||
332 | 61 | ubuntu = lp.distributions['ubuntu'] | ||
333 | 62 | series = ubuntu.getSeries(name_or_version=product_series(release)[1]) | ||
334 | 63 | |||
335 | 64 | real_threshold = None | ||
336 | 65 | if latest_date_created: | ||
337 | 66 | # Allow a grace period to cope with publications arriving out of | ||
338 | 67 | # order during long transactions. | ||
339 | 68 | real_threshold = latest_date_created - datetime.timedelta(hours=1) | ||
340 | 69 | |||
341 | 70 | if ppa: | ||
342 | 71 | archive, group, ppa_full_name = lpl_common.get_archive( | ||
343 | 72 | ppa, | ||
344 | 73 | lp, | ||
345 | 74 | False, | ||
346 | 75 | distribution=ubuntu | ||
347 | 76 | ) | ||
348 | 77 | else: | ||
349 | 78 | archive = ubuntu.main_archive | ||
350 | 79 | |||
352 | 80 | debug(f"Retrieving Launchpad publications since {real_threshold}") | 59 | debug(f"Retrieving Launchpad publications since {real_threshold}") |
353 | 81 | sources = archive.getPublishedSources(order_by_date=True, created_since_date=real_threshold, distro_series=series) | 60 | sources = archive.getPublishedSources(order_by_date=True, created_since_date=real_threshold, distro_series=series) |
354 | 82 | for s in sources: | 61 | for s in sources: |
355 | 62 | override_pocket = None | ||
356 | 83 | if s.pocket not in pockets: | 63 | if s.pocket not in pockets: |
357 | 84 | continue | 64 | continue |
358 | 85 | 65 | ||
359 | @@ -90,15 +70,16 @@ def update_cache(release, cache, ppa=None, latest_date_created=None): | |||
360 | 90 | src = s.source_package_name | 70 | src = s.source_package_name |
361 | 91 | src_ver = s.source_package_version | 71 | src_ver = s.source_package_version |
362 | 92 | src_component = None | 72 | src_component = None |
364 | 93 | if not ppa: | 73 | if ppa: |
365 | 94 | src_component = s.component_name | 74 | src_component = s.component_name |
366 | 75 | override_pocket = release_ppa_pocket(release, ppa) | ||
367 | 95 | 76 | ||
368 | 96 | binaries = s.getPublishedBinaries(active_binaries_only=False) | 77 | binaries = s.getPublishedBinaries(active_binaries_only=False) |
369 | 97 | for b in binaries: | 78 | for b in binaries: |
370 | 98 | bin_name = b.binary_package_name | 79 | bin_name = b.binary_package_name |
371 | 99 | bin_version = b.binary_package_version | 80 | bin_version = b.binary_package_version |
372 | 100 | bin_component = b.component_name | 81 | bin_component = b.component_name |
374 | 101 | pocket = b.pocket | 82 | pocket = b.pocket if not ppa else override_pocket |
375 | 102 | bin_arch = b.display_name.split(' ')[-1] | 83 | bin_arch = b.display_name.split(' ')[-1] |
376 | 103 | 84 | ||
377 | 104 | if src not in cache: | 85 | if src not in cache: |
378 | @@ -120,6 +101,33 @@ def update_cache(release, cache, ppa=None, latest_date_created=None): | |||
379 | 120 | 101 | ||
380 | 121 | return latest_date_created | 102 | return latest_date_created |
381 | 122 | 103 | ||
382 | 104 | def update_cache(release, cache, ppas=None, latest_date_created=None): | ||
383 | 105 | lp = lpl_common.connect(version='devel') | ||
384 | 106 | ubuntu = lp.distributions['ubuntu'] | ||
385 | 107 | series = ubuntu.getSeries(name_or_version=product_series(release)[1]) | ||
386 | 108 | |||
387 | 109 | real_threshold = None | ||
388 | 110 | if latest_date_created: | ||
389 | 111 | # Allow a grace period to cope with publications arriving out of | ||
390 | 112 | # order during long transactions. | ||
391 | 113 | real_threshold = latest_date_created - datetime.timedelta(hours=1) | ||
392 | 114 | |||
393 | 115 | if ppas: | ||
394 | 116 | for ppa_pocket in ppas: | ||
395 | 117 | ppa = ppa_pocket['ppa'] | ||
396 | 118 | archive, _, _ = lpl_common.get_archive( | ||
397 | 119 | ppa.split('/ubuntu')[0], | ||
398 | 120 | lp, | ||
399 | 121 | False, | ||
400 | 122 | distribution=ubuntu | ||
401 | 123 | ) | ||
402 | 124 | |||
403 | 125 | latest_date_created = get_binaries_rel(release, archive, real_threshold, series, cache, latest_date_created, ppa) | ||
404 | 126 | else: | ||
405 | 127 | latest_date_created = get_binaries_rel(release, ubuntu.main_archive, real_threshold, series, cache, latest_date_created) | ||
406 | 128 | |||
407 | 129 | return latest_date_created | ||
408 | 130 | |||
409 | 123 | 131 | ||
410 | 124 | def warn(message): | 132 | def warn(message): |
411 | 125 | """ print a warning message """ | 133 | """ print a warning message """ |
412 | @@ -186,8 +194,8 @@ def main(): | |||
413 | 186 | 194 | ||
414 | 187 | debug('UPDATING CACHE') | 195 | debug('UPDATING CACHE') |
415 | 188 | 196 | ||
418 | 189 | ppa = release_ppa(release) | 197 | ppas = release_ppas(release) |
419 | 190 | latest_date_created = update_cache(release, cache, ppa, latest_date_created) | 198 | latest_date_created = update_cache(release, cache, ppas, latest_date_created) |
420 | 191 | 199 | ||
421 | 192 | if latest_date_created is not None: | 200 | if latest_date_created is not None: |
422 | 193 | epoch = datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc) | 201 | epoch = datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc) |
423 | diff --git a/scripts/test_source_map.py b/scripts/test_source_map.py | |||
424 | index 8e99825..e05117c 100755 | |||
425 | --- a/scripts/test_source_map.py | |||
426 | +++ b/scripts/test_source_map.py | |||
427 | @@ -13,7 +13,7 @@ mock_subprojects = { | |||
428 | 13 | "packages": ["test/esm-fake-supported.txt"], | 13 | "packages": ["test/esm-fake-supported.txt"], |
429 | 14 | "name": "Ubuntu 01.01 ESM", | 14 | "name": "Ubuntu 01.01 ESM", |
430 | 15 | "codename": "Fabulous Fake", | 15 | "codename": "Fabulous Fake", |
432 | 16 | "ppa": "ubuntu-esm/esm-bar-security/ubuntu", | 16 | "ppas": [{"ppa": "ubuntu-esm/esm-bar-security/ubuntu", "pocket": "main"}], |
433 | 17 | "parent": "ubuntu/bar", | 17 | "parent": "ubuntu/bar", |
434 | 18 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", | 18 | "description": "Available with Ubuntu Pro: https://ubuntu.com/pro", |
435 | 19 | "stamp": 1618963200, | 19 | "stamp": 1618963200, |
LGTM, thanks!