Merge lp:~powersj/simplestreams/enable-tox into lp:simplestreams

[Joshua Powers]

[Joshua Powers (powersj)]

Here is full output on fresh yakkety container:
http://paste.ubuntu.com/21166479/

[Scott Moser (smoser)]

lets change the Makefile to use tools/sign-examples.

http://paste.ubuntu.com/21325350/ will install all dependencies as read from build-depends.

i'd like to maintain a difference between what is required to run 'tox' (which really shouldnt be more than python-tox) and what is required to build.

distro-info is obnoxious in that python-distro-info the pip package is not really useful without having the data (which is installed by distro-info-data).
I guess we only use that in the tools/ directory though.

[lp:~powersj/simplestreams/enable-tox]

441. By Joshua Powers on 2016-07-29

Broke out gnupg code as it its called in various places in the Makefile.

Removed the requirements file as it does not belong.

[Scott Moser (smoser)]

this looks good.
one small comment.

[Scott Moser (smoser)]

also, looks like you might need python-dev and / or python3-dev to pip install python-glanceclient. at least in my test i did, as i got a failure like

...
    x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fno-strict-aliasing -Wdate-time -D_FORTIFY_SOURCE=2 -g -fstack-protector-strong -Wformat -Werror=format-security -fPIC -DNETIFACES_VERSION=0.10.4 -DHAVE_GETIFADDRS=1 -DHAVE_GETNAMEINFO=1 -DHAVE_NETASH_ASH_H=1 -DHAVE_NETATALK_AT_H=1 -DHAVE_NETAX25_AX25_H=1 -DHAVE_NETECONET_EC_H=1 -DHAVE_NETIPX_IPX_H=1 -DHAVE_NETPACKET_PACKET_H=1 -DHAVE_LINUX_IRDA_H=1 -DHAVE_LINUX_ATM_H=1 -DHAVE_LINUX_LLC_H=1 -DHAVE_LINUX_TIPC_H=1 -DHAVE_LINUX_DN_H=1 -DHAVE_SOCKADDR_AT=1 -DHAVE_SOCKADDR_AX25=1 -DHAVE_SOCKADDR_IN=1 -DHAVE_SOCKADDR_IN6=1 -DHAVE_SOCKADDR_IPX=1 -DHAVE_SOCKADDR_UN=1 -DHAVE_SOCKADDR_ASH=1 -DHAVE_SOCKADDR_EC=1 -DHAVE_SOCKADDR_LL=1 -DHAVE_SOCKADDR_ATMPVC=1 -DHAVE_SOCKADDR_ATMSVC=1 -DHAVE_SOCKADDR_DN=1 -DHAVE_SOCKADDR_IRDA=1 -DHAVE_SOCKADDR_LLC=1 -DHAVE_PF_NETLINK=1 -I/usr/include/python2.7 -c netifaces.c -o build/temp.linux-x86_64-2.7/netifaces.o
    netifaces.c:1:20: fatal error: Python.h: No such file or directory
    compilation terminated.
    error: command 'x86_64-linux-gnu-gcc' failed with exit status 1

    ----------------------------------------
Command "/home/smoser/pub/venvs/smt/bin/python2 -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-mKbyFI/netifaces/setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --record /tmp/pip-H5gvlb-record/install-record.txt --single-version-externally-managed --compile --install-headers /home/smoser/pub/venvs/smt/include/site/python2.7/netifaces" failed with error code 1 in /tmp/pip-build-mKbyFI/netifaces/

this is definitely where i think pip falls apart. once you need things that aren'tpure python.

[lp:~powersj/simplestreams/enable-tox]

442. By Joshua Powers on 2016-08-01

Moved flake8 dependency to be more specifc to the test.

[Joshua Powers (powersj)]

What do you want to do about the test dependencies? Document them? This again was the list for the Ubuntu packages that need to be installed to run successfully.

> build-essential
> python-dev
> python-distro-info
> python-tox
> python3-dev

=== modified file 'Makefile'
--- Makefile	2014-07-15 21:37:26 +0000
+++ Makefile	2016-08-01 15:37:21 +0000
@@ -1,11 +1,5 @@
 TENV := ./tools/tenv
 
-PUBKEY := examples/keys/example.pub
-PUBKEYS := $(PUBKEY)
-SECKEY := examples/keys/example.sec
-
-example_sstream_files := $(wildcard examples/*/streams/v1/*.json)
-
 EXDATA_SIGN ?= 1
 ifeq ($(EXDATA_SIGN),1)
     EXDATA_SIGN_ARG := --sign
@@ -47,28 +41,10 @@
 exdata-query:
 	rsync -avz --delete --exclude "FILE_DATA_CACHE" --exclude ".bzr/*" cloud-images.ubuntu.com::uec-images/query/ exdata-query
 
-$(PUBKEY) $(SECKEY):
-	@mkdir -p $$(dirname "$(PUBKEY)") $$(dirname "$(SECKEY)")
-	$(TENV) gen-example-key $(PUBKEY) $(SECKEY)
-
-gnupg: gnupg/README
-
-gnupg/README: $(PUBKEYS) $(SECKEY)
-	rm -Rf gnupg
-	@umask 077 && mkdir -p gnupg
-	$(TENV) gpg --import $(SECKEY) >/dev/null 2>&1
-	for pubkey in $(PUBKEYS); do \
-	  $(TENV) gpg-trust-pubkey $$pubkey; done
-	@echo "this is used by $(TENV) as the gpg directory" > gnupg/README
-
-# this is less than ideal, but Make and ':' in targets do not work together
-# so instead of proper rules, we have this phoney rule that makes the
-# targets.  This would probably cause issue with -j.
+gnupg:
+	./tools/gnupg
+
 examples-sign: gnupg
-	@for f in $(example_sstream_files); do \
-		[ "$$f.gpg" -nt "$$f" -a "$${f%.json}.sjson" -nt "$$f" ] || \
-		{ echo "$(TENV) js2signed $$f" 1>&2; $(TENV) js2signed $$f; } || exit; \
-	done
-
+	./tools/example-sign
 
 .PHONY: check exdata/fake exdata/data exdata-query examples-sign test test2 test3 lint lint2 lint3

=== added file 'tools/example-sign'
--- tools/example-sign	1970-01-01 00:00:00 +0000
+++ tools/example-sign	2016-08-01 15:37:21 +0000
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+TOP_DIR=$(cd "$(dirname "${0}")"/.. && pwd)
+TENV=${TOP_DIR}/tools/tenv
+FILES=( ${TOP_DIR}/examples/*/streams/v1/*.json )
+
+if [ ! -f "${TOP_DIR}/gnupg/README" ]
+then
+    echo "Cannot sign: gpg key not setup. Run gnupg."
+    exit 2
+fi
+
+echo -e "\nGenerating signed files..."
+for f in "${FILES[@]}"
+do 
+    if [ "$f.gpg" -ot "$f" -o "${f%.json}.sjson" -ot "$f" ]
+    then 
+        echo "tenv js2signed $f" 1>&2;
+        $TENV js2signed "$f";
+    fi
+done

=== added file 'tools/gnupg'
--- tools/gnupg	1970-01-01 00:00:00 +0000
+++ tools/gnupg	2016-08-01 15:37:21 +0000
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+TOP_DIR=$(cd "$(dirname "${0}")"/.. && pwd)
+GPG_DIR=${TOP_DIR}/gnupg
+TENV=${TOP_DIR}/tools/tenv
+PUBKEY=${TOP_DIR}/examples/keys/example.pub
+SECKEY=${TOP_DIR}/examples/keys/example.sec
+
+if [ -f "$GPG_DIR"/README ]
+then
+    echo "Files exist, not regenerating." 
+    exit
+fi
+
+echo -e "\nSetting up GPG keys..."
+umask 077 && mkdir -p "$GPG_DIR"
+
+if [ ! -f "$PUBKEY" -o ! -f "$SECKEY" ]
+then
+    mkdir -p "$(dirname "$PUBKEY")"
+    out=$($TENV "${TOP_DIR}"/tools/gen-example-key "$PUBKEY" "$SECKEY") ||
+        { echo "Failed to generate keys: $out"; exit 2; }
+fi
+
+out=$($TENV gpg --import "$SECKEY" >/dev/null 2>&1 ) ||
+    { echo "Failed to import seckey: $out"; exit 2; }
+out=$("${TOP_DIR}"/tools/gpg-trust-pubkey "$PUBKEY") ||
+    { echo "Failed to import pubkey: $out"; exit 2; }
+
+echo "this is used by $TENV as the gpg directory" > "$GPG_DIR"/README
+

=== added file 'tox.ini'
--- tox.ini	1970-01-01 00:00:00 +0000
+++ tox.ini	2016-08-01 15:37:21 +0000
@@ -0,0 +1,24 @@
+[tox]
+envlist = py27, py3, py3-flake8
+recreate = True
+
+[testenv]
+commands = 
+    {toxinidir}/tools/gnupg
+    {toxinidir}/tools/example-sign
+    {envpython} -m nose {posargs:--with-coverage --cover-erase \
+        --cover-branches --cover-package=simplestreams --cover-inclusive \
+        tests} 
+deps =
+    coverage
+    mock
+    nose
+    python-glanceclient
+    python-keystoneclient 
+
+[testenv:py3-flake8]
+envdir = {toxworkdir}/py3
+deps = flake8
+commands = {envpython} -m flake8 {posargs:bin/ simplestreams/ tests/ tools/}
+ignore=H404,H405,H105,H301,H104,H403,H101
+exclude = .venv,.tox,dist,doc,*egg,.git,build,tools