Merge lp:~smoser/maas-images/lp1689557-build-squashfs-trusty into lp:maas-images

Proposed by Scott Moser
Status: Merged
Merged at revision: 376
Proposed branch: lp:~smoser/maas-images/lp1689557-build-squashfs-trusty
Merge into: lp:maas-images
Diff against target: 293 lines (+289/-0)
1 file modified
bin/img2squashfs (+289/-0)
To merge this branch: bzr merge lp:~smoser/maas-images/lp1689557-build-squashfs-trusty
Reviewer Review Type Date Requested Status
Lee Trager (community) Approve
Review via email: mp+323920@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Scott Moser (smoser) wrote :

This is a work in progress.
The goal of the branch is to allow creation of a squashfs image in the output from a build of trusty or precise (which do not have a .squashfs image).

Currently, what happens in those case is output contains the -root.tar.xz output.

The plan is to change meph2/stream.py 'create_version' to create a squashfs image if the input wasnt squashfs and the config indicated squashfs.

Revision history for this message
Lee Trager (ltrager) wrote :

Thanks for coding this up. I've been successfully using it with lp:~ltrager/maas-images/v3_squashfs_only Feel free to land otherwise I'll land it with my branch.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'bin/img2squashfs'
2--- bin/img2squashfs 1970-01-01 00:00:00 +0000
3+++ bin/img2squashfs 2017-05-11 14:54:13 +0000
4@@ -0,0 +1,289 @@
5+#!/bin/bash
6+
7+VERBOSITY=1
8+TEMP_D=""
9+VALID_FORMATS=( auto img-tar root-image root-image-gz root-tar squashfs-image
10+ dir )
11+
12+error() { echo "$@" 1>&2; }
13+fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
14+
15+Usage() {
16+ cat <<EOF
17+Usage: ${0##*/} [ options ] source-image output-squash-image
18+
19+ convert 'source-image' into a squashfs filesystem image.
20+
21+ source-image is a file in one of:
22+ ${VALID_FORMATS[*]}
23+
24+ output-squash-image will be a squashfs image.
25+
26+ options:
27+ -v | --verbose increase verbosity
28+ -f | --format F source-image is of format F. default: auto.
29+ must be one of:
30+ auto: determine based on file and name heuristics
31+ img-tar: tarball of root image (image file named .img)
32+ root-image: filesytem (ext[234] in a file)
33+ root-image-gz: root-image that is compressed with gzip
34+ root-tar: tarball of / (supports .tar.xz .tar.gz)
35+ squashfs-image: a squahsfs image (.squashfs)
36+ -O | --owner O change ownership of output to O (user:group)
37+
38+ Example:
39+ ${0##*/} --format=my.tar.gz my.squashfs
40+EOF
41+}
42+
43+bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; return 1; }
44+cleanup() {
45+ [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
46+}
47+
48+ddebug() {
49+ local level="$1"; shift;
50+ [ "${level}" -gt "${VERBOSITY}" ] && return
51+ error "$(date -R):" "$@"
52+}
53+
54+debug() {
55+ local level=${1}; shift;
56+ [ "${level}" -gt "${VERBOSITY}" ] && return
57+ error "${@}"
58+}
59+
60+inargs() {
61+ #inargs(needle, [haystack])
62+ # return true if needle is in haystack
63+ local needle="$1" i=""
64+ shift
65+ for i in "$@"; do
66+ [ "$i" = "$needle" ] && return 0
67+ done
68+ return 1
69+}
70+
71+get_img_from_tar() {
72+ # given 'img' (image.tar.gz), extract it and put the .img file in 'out'
73+ local img="$1" out="$2" tempd="$3"
74+ local mtmp=$(mktemp -d "${tempd}/img_from_tar.XXXXXX")
75+ tar -C "$mtmp" -Sxf "$img" ||
76+ { error "failed to extract $img"; return 1; }
77+ local f="" found=""
78+ for f in "${mtmp}/"*; do
79+ [ ! -f "$f" -o "${f%.img}" = "$f" ] && continue
80+ [ -z "$found" ] ||
81+ { error "multiple .img found in $img"; return 1; }
82+ found="$f"
83+ done
84+ [ -n "$found" ] || { error "no .img in $img"; return 1; }
85+ mv "$found" "$out" && rm -Rf "$mtmp"
86+}
87+
88+determine_format() {
89+ local input="$1" fmt="$2" fout=""
90+ inargs "$fmt" "${VALID_FORMATS[@]}" || {
91+ error "provided '--format=$fmt' not a valid format:" \
92+ "${VALID_FORMATS[*]}"
93+ return 1;
94+ }
95+ [ "$fmt" != "auto" ] && _RET="$fmt" && return 0
96+ if [ -d "$input" ]; then
97+ debug 1 "${input} is a directory";
98+ _RET="$dir"
99+ return
100+ fi
101+ fout=$(LANG=C file "$input") ||
102+ { error "failed: file $input"; return 1; }
103+ case "${fout#$input: }" in
104+ "gzip compressed"*)
105+ fout=$(zcat "$input" | file -)
106+ case "${fout#*: }" in
107+ POSIX\ tar*) fmt="img-tar";;
108+ *) fmt="root-image-gz";;
109+ esac
110+ ;;
111+ "POSIX tar"*) fmt="img-tar";;
112+ *ext[234]\ filesystem*) fmt="root-image";;
113+ *[Ss]quashfs*) fmt="squashfs-image";;
114+ *)
115+ # if the above failed (on trusty a .tar.gz file was reported
116+ # as a Minux file system) then try filename based heuristics
117+ case "$input" in
118+ *-root.t??|*-root.tar|*-root.tar.??) fmt="root-tar";;
119+ *.tar.gz|*.tgz|*.tar) fmt="img-tar";;
120+ *.gz) fmt="root-image-gz";;
121+ *.squashfs) fmt="squashfs-image";;
122+ *)
123+ error "WARN: file '$input' did not match name hueristics"
124+ fmt="root-image";;
125+ esac
126+ debug 1 "guessing $input is $fmt based on name and 'file' [$fout]";;
127+ esac
128+ debug 1 "determined format is $fmt"
129+ _RET="$fmt"
130+}
131+
132+get_usable_input() {
133+ # given 'input' in format 'fmt' return:
134+ # _RET_fmt: the format of _RET_PATH. either
135+ # fs-image : a mountable filesystem image.
136+ # dir : a directory
137+ # _RET_path: full path to a file in tempd
138+ local input="$1" fmt="$2" tempd="$3" cmd=""
139+ case "$fmt" in
140+ img-tar)
141+ _RET_fmt="fs-image"
142+ _RET_path="fs-image"
143+ get_img_from_tar "$input" "${tempd}/fs-image" "${tempd}"
144+ return;;
145+ root-image-gz)
146+ _RET_fmt="fs-image"
147+ _RET_path="$tempd/fs-image"
148+ zcat -c "$input" > "$tempd/fs-image.tmp" &&
149+ mv "$tempd/fs-image.tmp" "$tempd/fs-image"
150+ return;;
151+ root-image)
152+ _RET_fmt="fs-image"
153+ _RET_path="$tempd/fs-image"
154+ ln -s "$(readlink -f "$input")" "${tempd}/fs-image"
155+ return;;
156+ squashfs-image)
157+ _RET_fmt="squashfs-image"
158+ _RET_PATH="${tempd}/fs-image"
159+ ln -s "$(readlink -f "$input")" "${tempd}/fs-image"
160+ return;;
161+ root-tar)
162+ _RET_fmt="dir"
163+ _RET_path="${tempd}/fs-dir"
164+ mkdir "${_RET_path}"
165+ cmd=( tar -C "${_RET_path}" -xpSf "$input" \
166+ --numeric-owner --xattrs "--xattrs-include=*" )
167+ debug 1 "extracting $fmt with ${cmd[*]}"
168+ "${cmd[@]}" || {
169+ error "failed: extracting $fmt with ${cmd[*]}"
170+ return 1;
171+ }
172+ return
173+ ;;
174+ dir)
175+ _RET_fmt="dir"
176+ _RET_PATH="${tempd}/fs-dir"
177+ ln -s "$(readlink -f "$input")" "${tempd}/fs-dir"
178+ return;;
179+ *)
180+ error "Unknown format '$fmt'";
181+ return 1;;
182+ esac
183+}
184+
185+
186+dir2squashfs() {
187+ local src_d="$1" out="$2" owner="${3:-$(id -u):$(id -g)}"
188+ local tmpfile ret="" cmd=""
189+ tmpfile="${out}.${0##*/}.$$" || return
190+ cmd=( mksquashfs "$src_d" "$tmpfile" -xattrs -comp xz )
191+ ddebug 1 "starting: ${cmd[*]}"
192+ "${cmd[@]}"
193+ ret=$?
194+ ddebug 1 "finished: returned $ret"
195+ if [ $ret -eq 0 ]; then
196+ chown "$owner" "$tmpfile" ||
197+ { error "failed chown $owner $tmpfile"; return 1; }
198+ mv "$tmpfile" "$out" ||
199+ { error "failed to move file to $out"; return 1; }
200+ else
201+ rm -f "$tmpfile"
202+ error "mksquashfs failed [$ret]: ${cmd[*]}."
203+ fi
204+ return $ret
205+}
206+
207+fsimage2squashfs() {
208+ local src="$1" out="$2" owner="$3"
209+ local sudo="sudo"
210+ [ "$(id -u )" = "0" ] && sudo=""
211+ local cmd=""
212+ cmd=( mount-image-callback "$src" --
213+ "$0" dir2squashfs _MOUNTPOINT_ "$out" $owner )
214+ debug 1 "calling ${cmd[*]}"
215+ "${cmd[@]}"
216+}
217+
218+main() {
219+ local short_opts="hf:O:v"
220+ local long_opts="help,format:,owner:,verbose"
221+ local getopt_out=""
222+ getopt_out=$(getopt --name "${0##*/}" \
223+ --options "${short_opts}" --long "${long_opts}" -- "$@") &&
224+ eval set -- "${getopt_out}" ||
225+ { bad_Usage; return; }
226+
227+ local cur="" next="" input="" output="" owner="" fmt_in="auto" fmt=""
228+ while [ $# -ne 0 ]; do
229+ cur="$1"; next="$2";
230+ case "$cur" in
231+ -h|--help) Usage ; exit 0;;
232+ -f|--format) fmt_in="$next"; shift;;
233+ -O|--owner) owner="$next"; shift;;
234+ -v|--verbose) VERBOSITY=$((${VERBOSITY}+1));;
235+ --) shift; break;;
236+ esac
237+ shift;
238+ done
239+
240+ [ $# -eq 2 ] || { bad_Usage "must provide input and output."; return; }
241+ src_in="$1"
242+ output="$2"
243+
244+ TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX") ||
245+ { error "failed to make tempdir"; return 1; }
246+ trap cleanup EXIT
247+ local tempd="${TEMP_D}"
248+
249+ determine_format "${src_in}" "${fmt_in}" || return
250+ local fmt="${_RET}"
251+ debug 1 "format of '${src_in}' is '$fmt'"
252+
253+ case "$fmt" in
254+ dir|squashfs-image|root-tar) :;;
255+ *)
256+ [ "$(id -u)" = "0" ] || {
257+ error "Cannot convert $fmt to squashfs as non-root";
258+ return 1;
259+ }
260+ esac
261+ get_usable_input "${src_in}" "$fmt" "$tempd" || return
262+ local cfmt="${_RET_fmt}" tpath="${_RET_path}"
263+
264+ debug 1 "got format '$cfmt' at temp path '$tpath'"
265+ if [ "$cfmt" = "squashfs-image" ]; then
266+ local tmpf="$output.$$"
267+ cp "$src_in" "$tmpf" &&
268+ { [ -z "$owner" ] || chown "$owner" "$tmpf" ; } &&
269+ mv "$tmpf" "$output" || {
270+ rm -f "$tmpf"
271+ error "failed copying $src_in to $output"
272+ return 1
273+ }
274+ elif [ -f "$tpath" ]; then
275+ fsimage2squashfs "$tpath" "$output" "$owner" || return
276+ elif [ -d "$tpath" ]; then
277+ dir2squashfs "$tpath" "$output" || return
278+ else
279+ error "failed. path='$tpath' cfmt=${cfmt}"
280+ return 1
281+ fi
282+ error "output in $output. took ${SECONDS}s."
283+ return 0
284+}
285+
286+if [ "$1" = "dir2squashfs" ]; then
287+ shift
288+ dir2squashfs "$@"
289+else
290+ main "$@"
291+fi
292+
293+# vi: ts=4 expandtab

Subscribers

People subscribed via source and target branches