Merge ubuntu-cve-tracker:usn-summary into ubuntu-cve-tracker:master

[Mark Morlino]

* print the USN ID with the response
* Some changes required to allow for importing older USNs (present in database-all.json but not in database.json).
* Send the value of the isummary field from our database.json as the value of the summary field on the json data payload posted top the api
* Sort the USNs and Add an argument to start with a specific USN ID. This allows large batches to be restarted more easily without re-uploading stuff

[Alex Murray (alexmurray)]

LGTM!

review: Approve

[Steve Beattie (sbeattie)]

Approved additional commits.

review: Approve

diff --git a/scripts/publish-usn-to-website-api.py b/scripts/publish-usn-to-website-api.py
index 7ebf2bf..b67819b 100755
--- a/scripts/publish-usn-to-website-api.py
+++ b/scripts/publish-usn-to-website-api.py
@@ -100,6 +100,10 @@ parser.add_argument(
     "--no-upsert", action="store_true",
     help="No update after non-200 status on add, No add after non-200 status on update",
     )
+parser.add_argument(
+    "--startwith", action="store", type=str,
+    help='USN ID (without USN- prefix) to start with. Useful for resuming an interupted batch operation.',
+    )
 
 args = parser.parse_args()
 
@@ -109,6 +113,8 @@ if (args.action == 'add' or args.action == 'update') and not args.json:
     sys.exit(f"Error: --action {args.action} requires --json")
 if args.action == 'remove' and not args.json and not args.usns:
     sys.exit("Error: --action remove requires --json or --usns")
+if args.startwith and not re.match("\d{1,5}-\d{1,2}", args.startwith):
+    sys.exit("Error: --startwith must match '\d{1,5}-\d{1,2}'")
 
 client = httpbakery.Client(cookies=MozillaCookieJar(".login"))
 
@@ -128,91 +134,109 @@ if args.action == 'remove' and args.usns:
     sys.exit(0)
 
 # read in the json
-with open(args.json) as usn_json:
-    payload = json.load(usn_json).items()
+with open(args.json, 'r') as fh:
+    usn_json = json.load(fh)
+
+# split usn ids into tuples for sorting
+def usn_key(usn):
+    if not usn:
+        return ()
+    return tuple([int(n) for n in usn[0].split('-')])
+
+payload = sorted(usn_json.items(), key=usn_key)
 
-    for notice_id, notice in payload:
-        # format release_packages
-        release_packages = {}
+for notice_id, notice in payload:
 
-        for codename, packages in notice["releases"].items():
-            release_packages[codename] = []
+    if args.startwith and usn_key([notice_id]) < usn_key([args.startwith]):
+        continue
+
+    # format release_packages
+    release_packages = {}
+
+    for codename, packages in notice["releases"].items():
+        release_packages[codename] = []
+        if "sources" in packages:
             for name, info in packages["sources"].items():
                 release_packages[codename].append(
                     {
                         "name": name,
                         "version": info["version"],
-                        "description": info["description"],
+                        "description": info.get("description", ""),
                         "is_source": "true"
                     }
                 )
 
-            for name, info in packages["binaries"].items():
+        for name, info in packages["binaries"].items():
+            if "sources" in packages:
                 source_link, version_link = guess_binary_links(
                     name, info["version"], packages["sources"]
                 )
-                release_packages[codename].append(
-                    {
-                        "name": name,
-                        "version": info["version"],
-                        "is_source": "false",
-                        "source_link": source_link,
-                        "version_link": version_link,
-                    }
-                )
-        # format CVEs and references
-        cves = []
-        references = []
+            else:
+                source_link = ""
+                version_link = ""
+            release_packages[codename].append(
+                {
+                    "name": name,
+                    "version": info["version"],
+                    "is_source": "false",
+                    "source_link": source_link,
+                    "version_link": version_link,
+                }
+            )
+    # format CVEs and references
+    cves = []
+    references = []
+    if "cves" in notice:
         for reference in notice["cves"]:
-            if reference.startswith("CVE-"):
+            if re.match("CVE-\d{4}-\d+", reference) and not reference in cves:
                 cves.append(reference)
             else:
                 references.append(reference)
 
-        # Build json payload
-        json_data = {
-            "id": notice["id"],
-            "description": notice["description"],
-            "references": references,
-            "cves": cves,
-            "release_packages": release_packages,
-            "title": notice["title"],
-            "published": datetime.utcfromtimestamp(
-                notice["timestamp"]
-            ).isoformat(),
-            "summary": notice["summary"],
-            "instructions": notice.get("action"),
-        }
-        
-        # Build endpoint
+    # Build json payload
+    json_data = {
+        "id": notice["id"],
+        "description": notice["description"],
+        "references": references,
+        "cves": cves,
+        "release_packages": release_packages,
+        "title": notice["title"],
+        "published": datetime.utcfromtimestamp(
+            notice["timestamp"]
+        ).isoformat(),
+        "summary": notice.get("isummary", notice["title"]),
+        "instructions": notice.get("action", "In general, a standard system update will make all the necessary changes."),
+    }
+    
+    # Build endpoint
+    if args.action == 'add':
+        upsert = not args.no_upsert
+        http_method = 'POST'
+        endpoint = args.endpoint
+    elif args.action == 'update':
+        upsert = not args.no_upsert
+        http_method = 'PUT'
+        endpoint = f"{args.endpoint}/{notice['id']}"
+    elif args.action == 'remove':
+        upsert = False
+        http_method = 'DELETE'
+        endpoint = f"{args.endpoint}/{notice['id']}"
+        json_data = None
+
+    response = client.request( http_method, url=endpoint, json=json_data)
+    print(notice['id'], response, response.text[0:60])
+
+    if upsert and not re.match(r'^<Response \[2..\]>$',str(response)):
         if args.action == 'add':
-            upsert = not args.no_upsert
-            http_method = 'POST'
-            endpoint = args.endpoint
-        elif args.action == 'update':
-            upsert = not args.no_upsert
+            #print(f"{notice['id']} add failed, trying update")
             http_method = 'PUT'
             endpoint = f"{args.endpoint}/{notice['id']}"
-        elif args.action == 'remove':
-            upsert = False
-            http_method = 'DELETE'
-            endpoint = f"{args.endpoint}/{notice['id']}"
-            json_data = None
-
+        elif args.action == 'update':
+            #print(f"{notice['id']} update failed, trying add")
+            http_method = 'POST'
+            endpoint = args.endpoint
         response = client.request( http_method, url=endpoint, json=json_data)
-        print(response, response.text[0:60])
+        print(notice['id'], response, response.text[0:60])
 
-        if upsert and not re.match(r'^<Response \[2..\]>$',str(response)):
-            if args.action == 'add':
-                print("add failed, trying update")
-                http_method = 'PUT'
-                endpoint = f"{args.endpoint}/{notice['id']}"
-            elif args.action == 'update':
-                print("update failed, trying add")
-                http_method = 'POST'
-                endpoint = args.endpoint
-            response = client.request( http_method, url=endpoint, json=json_data)
-            print(response, response.text[0:60])
-
-        if args.stop and not re.match(r'^<Response \[2..\]>$',str(response)):
-            sys.exit(1)
+    if args.stop and not re.match(r'^<Response \[2..\]>$',str(response)):
+        sys.exit(1)