Merge ~ubuntu-docker-images/ubuntu-docker-images/+git/utils:golang-manifest-builder into ~ubuntu-docker-images/ubuntu-docker-images/+git/utils:master

[Sergio Durigan Junior]

This is the manifest builder script that I've been working on. It's ready to go into production, and Emilia from the security team has given her ACK.

[Christian Ehrhardt  (paelzer)]

Overal LGTM, just a minor question for one potentially unchecked error path.

[Sergio Durigan Junior (sergiodj)]

Thanks for the review, Christian.

[Bryce Harrington (bryce)]

A few super minor suggestions below but otherwise LGTM, and looks like Christian's question has been answered.

review: Approve

[Sergio Durigan Junior (sergiodj)]

Thanks, Bryce. I've addressed your comments and merged the branch.

diff --git a/golang-manifest-builder.py b/golang-manifest-builder.py
 new file mode 100755
index 0000000..583f1c6
--- /dev/null
+++ b/golang-manifest-builder.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+# This Python script can be used to generate a security manifest file
+# from a list of Golang modules, that can be obtained with (e.g.):
+#
+#   $ "go list -m all"
+#
+# The code has been written to be used with the telegraf and the
+# cortex OCI images in mind, so it is heavily tweaked to work with the
+# module list generated by these two packages.
+#
+# The idea is to use the information provided by pkg.go.dev and obtain
+# the git repository URL for each module.  Unfortunately, pkg.go.dev
+# does not offer an API, so we have to download the webpages and
+# scrape them.
+
+# Author: Sergio Durigan Junior <[e-mail]>
+
+from lxml import html
+import requests
+import os
+import sys
+
+# This script requires one argument, which is filename containing the
+# list of modules to be processed.
+if len (sys.argv) != 2:
+    print ("E: You must provide the filename containing the list of modules.",
+           file = sys.stderr)
+    sys.exit (1)
+
+# A dictionary that maps a Golang module name to a git tag prefix that
+# is used by the upstream.
+#
+# This is needed because some upstreams develop and release several
+# Golang modules from a single git repository, and, in order to
+# properly differentiate the release tags for each of these modules,
+# they use a "namespaced" (prefixed) tag.  Unfortunately, though, this
+# information is not present in the pkg.go.dev page, so we have to
+# keep a manual list here.
+TAG_PREFIX = {
+    "cloud.google.com/go/pubsub" : "pubsub/",
+    "cloud.google.com/go/bigquery" : "bigquery/",
+    "cloud.google.com/go/datastore" : "datastorage/",
+    "cloud.google.com/go/storage" : "storage/",
+    "cloud.google.com/go/bigtable" : "bigtable/",
+
+    "github.com/Azure/go-autorest/autorest" : "autorest/",
+    "github.com/Azure/go-autorest/autorest/adal" : "autorest/adal/",
+    "github.com/Azure/go-autorest/autorest/azure/auth" : "autorest/azure/auth/",
+    "github.com/Azure/go-autorest/autorest/azure/cli" : "autorest/azure/cli/",
+    "github.com/Azure/go-autorest/autorest/date" : "autorest/date/",
+    "github.com/Azure/go-autorest/autorest/to" : "autorest/to/",
+    "github.com/Azure/go-autorest/autorest/mocks" : "autorest/mocks/",
+    "github.com/Azure/go-autorest/autorest/validation" : "autorest/validation",
+    "github.com/Azure/go-autorest/logger" : "logger/",
+    "github.com/Azure/go-autorest/tracing" : "tracing/",
+
+    "github.com/hashicorp/consul/api" : "api/",
+    "github.com/hashicorp/consul/sdk" : "sdk/",
+
+    "go.elastic.co/apm/module/apmhttp" : "module/apmhttp/",
+    "go.elastic.co/apm/module/apmot" : "module/apmot/",
+    }
+
+with open(sys.argv[1], 'r') as f:
+    for line in f.readlines ():
+        # Grab the name and version of each module.
+        modinfo = line.split (' ')
+        modinfo[0] = modinfo[0].lstrip ().rstrip ().replace ('\n', '')
+        modinfo[1] = modinfo[1].lstrip ().rstrip ().replace ('\n', '')
+
+        # Grab the page.  We offer a retry mechanism in case of
+        # failure.
+        attempts = 0
+        while True:
+            page = requests.get ('https://pkg.go.dev/{}'.format (modinfo[0]))
+            tree = html.fromstring (page.content)
+
+            try:
+                repo = tree.xpath ('//div[@class="UnitMeta"]/a/text()')[0].replace ('\n', '').lstrip ().rstrip ()
+            except IndexError:
+                attempts = attempts + 1
+                if attempts < 3:
+                    continue
+
+                print ("E: Unable to obtain information for module '{}'".format (modinfo[0]),
+                       file = sys.stderr)
+
+                if sys.stdin.isatty ():
+                    print ("Would you like to retry? (Y/n) ",
+                           file = sys.stderr, end = '')
+                    retry = input ()
+                    if retry == 'n':
+                        break
+                    continue
+                else:
+                    # If the input is not coming from a tty, it likely
+                    # means we're building on docker/LP or some such.
+                    # In this case, we want to fail so that the user
+                    # can retry the whole build.
+                    sys.exit (1)
+            break
+
+        # Do we need to adjust the tag?
+        if modinfo[0] in TAG_PREFIX.keys ():
+            modinfo[1] = "{}{}".format (TAG_PREFIX[modinfo[0]], modinfo[1])
+
+        print ("{},{}".format (repo, modinfo[1]).replace ('\n', ''))