Merge ~dbungert/curtin:test-help into curtin:master

[Dan Bungert]

fixes for running the unit and integration tests

Update some out-of-date build and test infrastructure

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:cc552c4c98f336c377ac30e6ded5d387437c2a40
https://jenkins.canonical.com/server-team/job/curtin-ci/108/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/server-team/job/curtin-ci/nodes=metal-amd64/108/
    SUCCESS: https://jenkins.canonical.com/server-team/job/curtin-ci/nodes=metal-arm64/108/
    SUCCESS: https://jenkins.canonical.com/server-team/job/curtin-ci/nodes=metal-ppc64el/108/
    SUCCESS: https://jenkins.canonical.com/server-team/job/curtin-ci/nodes=metal-s390x/108/

Click here to trigger a rebuild:
https://jenkins.canonical.com/server-team/job/curtin-ci/108//rebuild

review: Approve (continuous-integration)

[Michael Hudson-Doyle (mwhudson)]

All this looks fine. Are you going to merge manually?

review: Approve

[Olivier Gayot (ogayot)]

A comment about the test helper script, but otherwise LGTM.

diff --git a/Makefile b/Makefile
index c04c4cf..5e8822a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,5 @@
 TOP := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 CWD := $(shell pwd)
-PYTHON2 ?= python2
 PYTHON3 ?= python3
 COVERAGE ?= 1
 DEFAULT_COVERAGEOPTS = --with-coverage --cover-erase --cover-branches --cover-package=curtin --cover-inclusive 
@@ -16,11 +15,11 @@ target_dirs ?= curtin tests tools
 build:
 
 bin/curtin: curtin/pack.py tools/write-curtin
-	$(PYTHON) tools/write-curtin bin/curtin
+	$(PYTHON3) tools/write-curtin bin/curtin
 
 check: unittest
 
-style-check: pep8 pyflakes pyflakes3
+style-check: pep8 pyflakes3
 
 coverage: coverageopts ?= $(DEFAULT_COVERAGEOPTS)
 coverage: unittest
@@ -28,26 +27,15 @@ coverage: unittest
 pep8:
 	@$(CWD)/tools/run-pep8
 
-pyflakes:
-	$(PYTHON2) -m pyflakes $(target_dirs)
-
-pyflakes3:
+pyflakes pyflakes3:
 	$(PYTHON3) -m pyflakes $(target_dirs)
 
-pylint:
-	$(PYTHON2) -m pylint $(pylintopts) $(target_dirs)
-
-pylint3:
+pylint pylint3:
 	$(PYTHON3) -m pylint $(pylintopts) $(target_dirs)
 
-unittest2:
-	$(PYTHON2) -m nose $(coverageopts) $(noseopts) tests/unittests
-
-unittest3:
+unittest unittest3:
 	$(PYTHON3) -m nose $(coverageopts) $(noseopts) tests/unittests
 
-unittest: unittest2 unittest3
-
 schema-validate:
 	@$(CWD)/tools/schema-validate-storage
 
@@ -55,7 +43,7 @@ docs: check-doc-deps
 	make -C doc html
 
 check-doc-deps:
-	@which sphinx-build && $(PYTHON) -c 'import sphinx_rtd_theme' || \
+	@which sphinx-build && $(PYTHON3) -c 'import sphinx_rtd_theme' || \
 		{ echo "Missing doc dependencies. Install with:"; \
 		  pkgs="python3-sphinx-rtd-theme python3-sphinx"; \
 		  echo sudo apt-get install -qy $$pkgs ; exit 1; }
diff --git a/test-requirements.txt b/test-requirements.txt
index 1970d03..0d93c02 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,3 +1,6 @@
+pip
+wheel
+virtualenv
 jsonschema
 mock
 nose
diff --git a/tests/integration/test_block_meta.py b/tests/integration/test_block_meta.py
index d701ca0..508847b 100644
--- a/tests/integration/test_block_meta.py
+++ b/tests/integration/test_block_meta.py
@@ -168,8 +168,7 @@ class StorageConfigBuilder:
 
     def add_image(self, *, path, size, create=False, **kw):
         if create:
-            with open(path, "wb") as f:
-                f.write(b"\0" * int(util.human2bytes(size)))
+            util.subp(['truncate', '-s', str(size), path])
         action = self._add(type='image', path=path, size=size, **kw)
         self.cur_image = action['id']
         return action
diff --git a/tools/vmtest-system-setup b/tools/vmtest-system-setup
index 4596df5..c34e9aa 100755
--- a/tools/vmtest-system-setup
+++ b/tools/vmtest-system-setup
@@ -11,27 +11,37 @@ case "$(uname -m)" in
   s390x) qemu="qemu-system-s390x";;
 esac
 
-get_python_apt() {
-    [[ "$1" < "21.04" ]] && echo python-apt
-    [[ "$1" > "16.04" ]] && echo python3-apt
-}
-
 DEPS=(
+  build-essential
   cloud-image-utils
+  git
   make
   net-tools
+  pep8
   python3
+  python3-apt
   python3-attr
+  python3-coverage
   python3-jsonschema
   python3-nose
+  python3-oauthlib
+  python3-pip
+  python3-pyflakes
+  python3-pytest
+  python3-pyudev
   python3-simplestreams
+  python3-wheel
   python3-yaml
-  $(get_python_apt "$(lsb_release -sr)")
+  lvm2
+  ntfs-3g
   ovmf
+  parted
   simplestreams
   $qemu
   ubuntu-cloudimage-keyring
   tgt
+  tox
+  wget
 )
 
 apt_get() {