Merge ~ubuntu-docker-images/ubuntu-docker-images/+git/utils:software-versions-image-extraction into ~ubuntu-docker-images/ubuntu-docker-images/+git/utils:master

[Bryce Harrington]

This implements the remaining functionality for looking up version info from the docker images. The text table output is getting a bit dense, but here's how it looks now:

Software focal hirsute
postgres: 12.6-0ubuntu0.20.04.1 12.6-0ubuntu0.20.04.1 (12.6) 13.2-1 ??? (13.2)
memcached: 1.5.22-2 1.5.22-2ubuntu0.1 (1.5.22) 1.6.9+dfsg-1 1.6.9+dfsg-1 (1.6.9)
apache2: 2.4.41-4ubuntu3.1 2.4.41-4ubuntu3.1 (2.4.41) 2.4.46-4ubuntu1 ??? (2.4.46)
mysql: 8.0.23-0ubuntu0.20.04.1 8.0.23-0ubuntu0.20.04.1 (8.0.23) 8.0.23-3ubuntu1 8.0.23-3build1 (8.0.23)
redis: 5:5.0.7-2 5:5.0.7-2 (5.0.7) 5:6.0.11-1 5:6.0.11-1 (6.0.11)
nginx: 1.18.0-0ubuntu1 1.18.0-0ubuntu1 (1.18.0) 1.18.0-6ubuntu8 1.18.0-6ubuntu4 (1.18.0)

(Note it's already telling us there are update opportunities for focal's memcached and hirsute's nginx.)

And check out the JSON report:

  https://paste.ubuntu.com/p/qmDxnRhKcM/

You'll note a huge amount of duplicate code between the text and json reports. I haven't decided whether to refactor that better, or just drop the text table -- as soon as we add another Ubuntu release to the mix the width of this table will become too unwieldy -- but whichever way, I'll leave that to a future MP.

You'll also note I'm still using hard-coded lists of images rather than using list-all-images.sh since I need to have some associated data. Refactoring this is also planned but left for a future MP.

[Athos Ribeiro (athos-ribeiro)]

Nice work! LGTM

I have only one question: should we either remove the pulled images or add an option to do so? If this script is running constantly in some CI environment, the internal docker storage may grow considerably depending on the frequency that new images get released.

In other words, would it be worth to sacrifice network data consumption in a CI environment (always pulling images even when no new images were pushed to the registries) to possibly avoid storage issues in the future?

Alternatively, we could keep only the latest versions (the one holding the tag) of each image in the host running the script.

review: Approve

[Bryce Harrington (bryce)]

Thanks for the review!

That's a good thought about cleanup, I hadn't thought that far ahead but you're right. For what I plan to use this for, it would be helpful to retain previously pulled images in a cache, however over time this would for sure build up. That maybe could be dealt with by a weekly or monthly purge and refresh, but perhaps there are better solutions out there. I've added a trello card for myself to track this:

  https://trello.com/c/dYw9NgO2/3933-software-versions-cache-control-of-pulled-images

[Bryce Harrington (bryce)]

I've landed this change to utils master branch.

diff --git a/README.software-versions.md b/README.software-versions.md
new file mode 100644
index 0000000..1eec933
--- /dev/null
+++ b/README.software-versions.md
@@ -0,0 +1,20 @@
+Software Versions
+=================
+
+software-versions.sh is used to review what versions of software
+packages provided in the Ubuntu OCI images, returning results either as
+JSON data or as a formatted text table.
+
+Currently, this only displays OCI images that ship debian-based
+software.
+
+
+Requirements
+------------
+
+You have to have the following programs installed in your system in
+order to run software-versions.sh:
+
+- distro-info
+- docker
+- rmadison
diff --git a/software-versions.sh b/software-versions.sh
index 1bb3125..b10fbdb 100755
--- a/software-versions.sh
+++ b/software-versions.sh
@@ -93,7 +93,7 @@ upstream_version() {
 ## @param package_name: Name of the source or binary package to look up.
 ## @param distro_codename: Ubuntu release name to check.
 ## @param type: Lookup either "source" or "binary" packages.
-package_version() {
+package_version_in_archive() {
     local package_name="${1}"
     local distro_codename="${2}"
     local type="${3:-source}"
@@ -107,25 +107,57 @@ package_version() {
 	suites="${distro_codename},${distro_codename}-updates,${distro_codename}-security"
     fi
 
-    local source_or_binary
     if [ "${type}" = "source" ]; then
-	source_or_binary="-a \"source\""
-    fi
-
-    rmadison \
-	"${source_or_binary}" \
-	-s "${suites}" \
-	"${package_name}" \
-	| cut -d\| -f2 \
+	rmadison -a "source" -s "${suites}" "${package_name}"
+    else
+	rmadison -s "${suites}" "${package_name}"
+    fi  | cut -d\| -f2 \
 	| sort --version-sort \
 	| tail -n 1 \
 	| xargs
 }
 
-source_package() {
-    local template v M N
+## Look up version of binary package in a Docker OCI image.
+##
+## @param image_id: Image to query from.
+## @param package_name: Name of the binary package to look up.
+package_version_in_image() {
+    local image_id binary_package_name
+    image_id="${1}"
+    binary_package_name="${2}"
+
+    [ -n "${image_id}" ] || return 1
+    [ -n "${binary_package_name}" ] || return 1
+
+    version=$(docker run --rm \
+		     --name "${binary_package_name}-version-check" \
+		     "${image_id}" \
+		     apt-cache policy "${binary_package_name}" \
+		  | grep "Installed:" | cut -d: -f2- | xargs) \
+	|| ( warn "Could not get ${image_id} from docker"; return 1 )
+
+    echo "${version}"
+}
+
+
+#########################
+### Software Versions ###
+#########################
+
+## Expand a package name template with the software version number.
+##
+## The template syntax uses {M}, {N}, and {M.N} as the variable
+## placeholders for the version number.
+##
+## @param template:  String template to be filled in.
+## @param v:  The 'M.N' version number to substitute.
+substitute_version() {
+    declare -a v
+    local template M N
     template="${1}"
-    v=( "${2//./ }" )
+
+    # shellcheck disable=SC2206
+    v=( ${2//./ } )
     M="${v[0]}"
     N="${v[1]}"
 
@@ -135,11 +167,6 @@ source_package() {
 	sed -e "s/{M.N}/${M}.${N}/g"
 }
 
-
-#########################
-### Software Versions ###
-#########################
-
 ## Returns the M.N version from a package version.
 ##
 ## We define "software version" to mean the initial two values in the
@@ -150,16 +177,23 @@ source_package() {
 ## @param str version: The package version to extract M.N from.
 ## @stdout: The M.N version string.
 software_version() {
-    local version v version_major version_minor
+    declare -a v
+    local version version_major version_minor
     version=$(upstream_version "${1}")
 
-    v=( "${version//./ }" )
+    # shellcheck disable=SC2206
+    v=( ${version//./ } )
     version_major="${v[0]}"
     version_minor="${v[1]}"
 
-    echo "${version_major}.${version_minor}"
+    if [ -z "${version_minor}" ]; then
+	echo "${version_major}"
+    else
+	echo "${version_major}.${version_minor}"
+    fi
 }
 
+
 ############
 ### Main ###
 ############
@@ -175,13 +209,13 @@ main() {
 
     # Debian packages in focal
     declare -A SOFTWARE=(
-	# Software    Source                               Binary
-	[apache2]="   apache{M}                            apache2"
-	[memcached]=" memcached                            memcached"
-	[mysql]="     mysql-{M.N}                          mysql-server"
-	[nginx]="     nginx                                nginx"
-	[postgres]="  postgresql-{M}                       postgresql"
-	[redis]="     redis                                redis-server"
+	# Software    Source          Metapackage         Installed-Binary
+	[apache2]="   apache2         apache2             apache2"
+	[memcached]=" memcached       memcached           memcached"
+	[mysql]="     mysql-{M.N}     mysql-server        mysql-server-core-{M.N}"
+	[nginx]="     nginx           nginx               nginx-common"
+	[postgres]="  postgresql-{M}  postgresql          postgresql-{M}"
+	[redis]="     redis           redis-server        redis-server"
     )
 
     declare -ar CODENAMES=(
@@ -197,20 +231,30 @@ main() {
 	    printf "%s "    "${CODENAMES[-1]}"
 	    printf "\n"
 	    for software_name in "${!SOFTWARE[@]}"; do
-		local RECORD=() source_package_template binary_package_name
+		local RECORD=() source_package_template
+		local meta_package_name binary_package_name
 
 		source_package_template=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $1}')
-		binary_package_name=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $2}')
+		meta_package_name=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $2}')
+		binary_package_template=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $3}')
 
 		local codename
 		for codename in "${CODENAMES[@]}"; do
-		    local binary_package_version software_version source_package version version_upstream
-		    binary_package_version=$(package_version "${binary_package_name}" "${codename}" "binary" )
-		    software_version=$(software_version "${binary_package_version}")
-		    source_package=$(source_package "${source_package_template}" "${software_version}")
-		    version=$(package_version "${source_package}" "${codename}" "source")
+		    local meta_package_version mn source_package_name image_id
+		    local version version_in_image version_upstream distro_release
+		    meta_package_version=$(package_version_in_archive "${meta_package_name}" "${codename}" "binary" )
+		    mn=$(software_version "${meta_package_version}")
+		    source_package_name=$(substitute_version "${source_package_template}" "${mn}")
+		    binary_package_name=$(substitute_version "${binary_package_template}" "${mn}")
+		    version=$(package_version_in_archive "${source_package_name}" "${codename}" "source")
 		    version_upstream=$(upstream_version "${version}")
-		    RECORD+=("${version} (${version_upstream})")
+
+		    distro_release=$(distro-info --series "${codename}" --release | cut -d' ' -f1)
+		    image_id="ubuntu/${software_name}:${mn}-${distro_release}_edge"
+		    version_in_image="???"
+		    docker image pull "${image_id}" 2>/dev/null >/dev/null \
+			&& version_in_image=$(package_version_in_image "${image_id}" "${binary_package_name}")
+		    RECORD+=("${version} <${version_in_image}> (${version_upstream})")
 		done
 
 		printf "%-25s " "${software_name}:"
@@ -224,25 +268,39 @@ main() {
 	    echo "["
 	    local num_software=${#SOFTWARE[@]}
 	    for software_name in "${!SOFTWARE[@]}"; do
-		local RECORD=() source_package_template binary_package_name
-		source_package_template=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $1}')
-		binary_package_name=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $2}')
+		local RECORD=() source_package_template
+		local meta_package_name binary_package_name
+
 		echo "    {"
 		echo "	\"software_name\":            \"${software_name}\","
 
+		source_package_template=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $1}')
+		meta_package_name=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $2}')
+		binary_package_template=$(echo "${SOFTWARE[${software_name}]}" | awk '{print $3}')
+
 		local codename
 		local num_codenames=${#CODENAMES[@]}
 		for codename in "${CODENAMES[@]}"; do
-		    local binary_package_version software_version source_package version version_upstream
-		    binary_package_version=$(package_version "${binary_package_name}" "${codename}" "binary" )
-		    software_version=$(software_version "${binary_package_version}")
-		    source_package=$(source_package "${source_package_template}" "${software_version}")
-		    version=$(package_version "${source_package}" "${codename}" "source")
+		    local meta_package_version mn source_package_name image_id
+		    local version version_in_image version_upstream distro_release
+		    meta_package_version=$(package_version_in_archive "${meta_package_name}" "${codename}" "binary" )
+		    mn=$(software_version "${meta_package_version}")
+		    source_package_name=$(substitute_version "${source_package_template}" "${mn}")
+		    binary_package_name=$(substitute_version "${binary_package_template}" "${mn}")
+		    version=$(package_version_in_archive "${source_package_name}" "${codename}" "source")
 		    version_upstream=$(upstream_version "${version}")
+
+		    distro_release=$(distro-info --series "${codename}" --release | cut -d' ' -f1)
+		    image_id="ubuntu/${software_name}:${mn}-${distro_release}_edge"
+		    version_in_image="???"
+		    docker image pull "${image_id}" 2>/dev/null >/dev/null \
+			&& version_in_image=$(package_version_in_image "${image_id}" "${binary_package_name}")
+
 		    echo "	\"${codename}\": {
-		\"source_package\":   \"${source_package}\",
-		\"software_version\": \"${software_version}\",
+		\"source_package\":   \"${source_package_name}\",
+		\"software_version\": \"${mn}\",
 		\"version\":          \"${version}\",
+		\"version_in_image\": \"${version_in_image}\",
 		\"version_upstream\": \"${version_upstream}\""
 		    if [ "${num_codenames}" = "1" ]; then
 			echo "		}"