Merge lp:~chipaca/snappy/dynamicExclusion into lp:~snappy-dev/snappy/snappy-moved-to-github

[John Lenton]

Introducing .snapignore, which is made of lines of regexps and not globs.

[lp:~chipaca/snappy/dynamicExclusion]

618. By John Lenton on 2015-07-29

addressed issues found in review

Unmerged revisions

=== modified file 'debian/golang-snappy-dev.install'
--- debian/golang-snappy-dev.install	2015-03-18 19:26:18 +0000
+++ debian/golang-snappy-dev.install	2015-07-29 09:50:51 +0000
@@ -1,1 +1,2 @@
+/usr/bin/xgettext-go
 /usr/share/gocode

=== modified file 'debian/rules'
--- debian/rules	2015-07-01 07:48:19 +0000
+++ debian/rules	2015-07-29 09:50:51 +0000
@@ -4,6 +4,7 @@
 #export DH_VERBOSE=1
 export DH_OPTIONS
 export DH_GOPKG := launchpad.net/snappy
+#export DEB_BUILD_OPTIONS=nocheck
 
 %:
 	dh $@ --buildsystem=golang --with=golang --fail-missing --with systemd
@@ -30,6 +31,10 @@
 		--no-enable \
 		-pubuntu-snappy \
 		snappy-autopilot.service
+	# enable wait4network
+	dh_systemd_enable \
+		-pubuntu-snappy \
+		snappy-wait4network.service
 
 override_dh_systemd_start:
 	# start boot-ok
@@ -49,6 +54,10 @@
 		--no-start \
 		-pubuntu-snappy \
 		snappy-autopilot.service
+	# start wait4network
+	dh_systemd_start \
+		-pubuntu-snappy \
+		snappy-wait4network.service
 
 override_dh_auto_build:
 	# this will update the i18n stuff

=== added file 'debian/snappy-wait4network.service'
--- debian/snappy-wait4network.service	1970-01-01 00:00:00 +0000
+++ debian/snappy-wait4network.service	2015-07-29 09:50:51 +0000
@@ -0,0 +1,9 @@
+[Unit]
+Description=Wait for network
+After=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+TimeoutStartSec=0
+ExecStart=/usr/bin/wait4network http://start.ubuntu.com/connectivity-check.html 4589f42e1546aa47ca181e5d949d310b 1s 5s

=== modified file 'debian/ubuntu-snappy-cli.install'
--- debian/ubuntu-snappy-cli.install	2015-07-01 07:47:55 +0000
+++ debian/ubuntu-snappy-cli.install	2015-07-29 09:50:51 +0000
@@ -1,4 +1,5 @@
 /usr/bin/snappy
+/usr/bin/wait4network
 data/completion/snappy /usr/share/bash-completion/completions/
 # i18n stuff
 ../../share /usr

=== modified file 'snappy/build.go'
--- snappy/build.go	2015-07-23 11:05:37 +0000
+++ snappy/build.go	2015-07-29 09:50:51 +0000
@@ -51,7 +51,7 @@
 //
 // Please resist the temptation of optimizing the regexp by grouping
 // things by hand. People will find it unreadable enough as it is.
-var shouldExclude = regexp.MustCompile(strings.Join([]string{
+var shouldExcludeDefault = regexp.MustCompile(strings.Join([]string{
 	`\.snap$`, // added
 	`\.click$`,
 	`^\..*\.sw.$`,
@@ -82,8 +82,59 @@
 	`^_MTN$`,
 	`^_darcs$`,
 	`^{arch}$`,
+	`^\.snapignore$`,
 }, "|")).MatchString
 
+// fake static function variables
+type keep struct {
+	basedir string
+	exclude func(string) bool
+}
+
+func (k *keep) shouldExclude(basedir string, file string) bool {
+	if basedir == k.basedir {
+		if k.exclude == nil {
+			return false
+		}
+		return k.exclude(file)
+	}
+
+	k.basedir = basedir
+	k.exclude = nil
+
+	f, err := os.Open(filepath.Join(basedir, ".snapignore"))
+	if err != nil {
+		return false
+	}
+
+	scanner := bufio.NewScanner(f)
+	var lines []string
+	for scanner.Scan() {
+		line := scanner.Text()
+		if _, err := regexp.Compile(line); err != nil {
+			// not a regexp
+			line = regexp.QuoteMeta(line)
+		}
+		lines = append(lines, line)
+	}
+
+	fullRegex := strings.Join(lines, "|")
+	exclude, err := regexp.Compile(fullRegex)
+	if err == nil {
+		k.exclude = exclude.MatchString
+
+		return k.exclude(file)
+	}
+
+	panic(fmt.Sprintf("|-composition of valid regexps is invalid?!? Please report this bug: %#v", fullRegex))
+}
+
+var shouldExcludeDynamic = new(keep).shouldExclude
+
+func shouldExclude(basedir string, file string) bool {
+	return shouldExcludeDefault(file) || shouldExcludeDynamic(basedir, file)
+}
+
 // small helper that return the architecture or "multi" if its multiple arches
 func debArchitecture(m *packageYaml) string {
 	switch len(m.Architectures) {
@@ -375,7 +426,7 @@
 			return errin
 		}
 
-		if shouldExclude(filepath.Base(path)) {
+		if shouldExclude(sourceDir, filepath.Base(path)) {
 			if info.IsDir() {
 				return filepath.SkipDir
 			}

=== modified file 'snappy/build_test.go'
--- snappy/build_test.go	2015-06-08 16:26:25 +0000
+++ snappy/build_test.go	2015-07-29 09:50:51 +0000
@@ -356,6 +356,28 @@
 	c.Check(string(out), Matches, `(?m)Only in \S+: \.bzr`)
 }
 
+func (s *SnapTestSuite) TestExcludeDynamicFalseIfNoSnapignore(c *C) {
+	basedir := c.MkDir()
+	c.Check(shouldExcludeDynamic(basedir, "foo"), Equals, false)
+}
+
+func (s *SnapTestSuite) TestExcludeDynamicWorksIfSnapignore(c *C) {
+	basedir := c.MkDir()
+	c.Assert(ioutil.WriteFile(filepath.Join(basedir, ".snapignore"), []byte("foo\nb.r\n"), 0644), IsNil)
+	c.Check(shouldExcludeDynamic(basedir, "foo"), Equals, true)
+	c.Check(shouldExcludeDynamic(basedir, "bar"), Equals, true)
+	c.Check(shouldExcludeDynamic(basedir, "bzr"), Equals, true)
+	c.Check(shouldExcludeDynamic(basedir, "baz"), Equals, false)
+}
+
+func (s *SnapTestSuite) TestExcludeDynamicWeirdRegexps(c *C) {
+	basedir := c.MkDir()
+	c.Assert(ioutil.WriteFile(filepath.Join(basedir, ".snapignore"), []byte("*hello\n"), 0644), IsNil)
+	// note “*hello” is not a valid regexp, so will be taken literally (not globbed!)
+	c.Check(shouldExcludeDynamic(basedir, "ahello"), Equals, false)
+	c.Check(shouldExcludeDynamic(basedir, "*hello"), Equals, true)
+}
+
 func (s *SnapTestSuite) TestBuildSimpleOutputDir(c *C) {
 	sourceDir := makeExampleSnapSourceDir(c, `name: hello
 version: 1.0.1

=== modified file 'snappy/click.go'
--- snappy/click.go	2015-07-23 11:05:37 +0000
+++ snappy/click.go	2015-07-29 09:50:51 +0000
@@ -444,6 +444,7 @@
 			StopTimeout: time.Duration(service.StopTimeout),
 			AaProfile:   aaProfile,
 			IsFramework: m.Type == pkg.TypeFramework,
+			IsNetworked: service.Ports != nil && len(service.Ports.External) > 0,
 			BusName:     service.BusName,
 			UdevAppName: udevPartName,
 		}), nil

=== modified file 'snappy/click_test.go'
--- snappy/click_test.go	2015-07-23 11:05:37 +0000
+++ snappy/click_test.go	2015-07-29 09:50:51 +0000
@@ -1231,6 +1231,7 @@
 WantedBy=multi-user.target
 `
 	expectedServiceAppWrapper = fmt.Sprintf(expectedServiceWrapperFmt, "After=ubuntu-snappy.frameworks.target\nRequires=ubuntu-snappy.frameworks.target", ".canonical", "canonical", "\n", helpers.UbuntuArchitecture())
+	expectedNetAppWrapper     = fmt.Sprintf(expectedServiceWrapperFmt, "After=ubuntu-snappy.frameworks.target\nRequires=ubuntu-snappy.frameworks.target\nAfter=snappy-wait4network.service\nRequires=snappy-wait4network.service", ".canonical", "canonical", "\n", helpers.UbuntuArchitecture())
 	expectedServiceFmkWrapper = fmt.Sprintf(expectedServiceWrapperFmt, "Before=ubuntu-snappy.frameworks.target\nAfter=ubuntu-snappy.frameworks-pre.target\nRequires=ubuntu-snappy.frameworks-pre.target", "", "", "BusName=foo.bar.baz\nType=dbus", helpers.UbuntuArchitecture())
 )
 
@@ -1253,6 +1254,26 @@
 	c.Assert(generatedWrapper, Equals, expectedServiceAppWrapper)
 }
 
+func (s *SnapTestSuite) TestSnappyGenerateSnapServiceAppWrapperWithExternalPort(c *C) {
+	service := ServiceYaml{
+		Name:        "xkcd-webserver",
+		Start:       "bin/foo start",
+		Stop:        "bin/foo stop",
+		PostStop:    "bin/foo post-stop",
+		StopTimeout: DefaultTimeout,
+		Description: "A fun webserver",
+		Ports:       &Ports{External: map[string]Port{"foo": Port{}}},
+	}
+	pkgPath := "/apps/xkcd-webserver.canonical/0.3.4/"
+	aaProfile := "xkcd-webserver.canonical_xkcd-webserver_0.3.4"
+	m := packageYaml{Name: "xkcd-webserver",
+		Version: "0.3.4"}
+
+	generatedWrapper, err := generateSnapServicesFile(service, pkgPath, aaProfile, &m)
+	c.Assert(err, IsNil)
+	c.Assert(generatedWrapper, Equals, expectedNetAppWrapper)
+}
+
 func (s *SnapTestSuite) TestSnappyGenerateSnapServiceFmkWrapper(c *C) {
 	service := ServiceYaml{
 		Name:        "xkcd-webserver",

=== modified file 'snappy/firstboot.go'
--- snappy/firstboot.go	2015-07-13 14:19:52 +0000
+++ snappy/firstboot.go	2015-07-29 09:50:51 +0000
@@ -28,6 +28,7 @@
 
 	"launchpad.net/snappy/helpers"
 	"launchpad.net/snappy/pkg"
+	"launchpad.net/snappy/systemd"
 
 	"gopkg.in/yaml.v2"
 )
@@ -135,7 +136,15 @@
 	ethfile := filepath.Join(ethdir, eth)
 	data := fmt.Sprintf("auto %[1]s\n\nallow-hotplug %[1]s\niface %[1]s inet dhcp\n", eth)
 
-	return helpers.AtomicWriteFile(ethfile, []byte(data), 0644)
+	if err := helpers.AtomicWriteFile(ethfile, []byte(data), 0644); err != nil {
+		return err
+	}
+
+	if _, err := systemd.SystemctlCmd("restart", "networking", "--no-block"); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func firstBootHasRun() bool {

=== modified file 'snappy/firstboot_test.go'
--- snappy/firstboot_test.go	2015-07-13 15:54:24 +0000
+++ snappy/firstboot_test.go	2015-07-29 09:50:51 +0000
@@ -28,6 +28,7 @@
 	. "gopkg.in/check.v1"
 
 	"launchpad.net/snappy/pkg"
+	"launchpad.net/snappy/systemd"
 )
 
 type fakePart struct {
@@ -54,11 +55,21 @@
 	oemConfig map[string]interface{}
 	globs     []string
 	ethdir    string
+	sctlargs  []string
+	sctlerr   error
 }
 
 var _ = Suite(&FirstBootTestSuite{})
 
+func (s *FirstBootTestSuite) systemctl(args ...string) (out []byte, err error) {
+	s.sctlargs = args
+	return nil, s.sctlerr
+}
+
 func (s *FirstBootTestSuite) SetUpTest(c *C) {
+	s.sctlerr = nil
+	s.sctlargs = nil
+	systemd.SystemctlCmd = s.systemctl
 	stampFile = filepath.Join(c.MkDir(), "stamp")
 
 	configMyApp := make(SystemConfig)
@@ -150,6 +161,20 @@
 	c.Assert(err, IsNil)
 	c.Check(string(bs), Equals, "auto eth42\n\nallow-hotplug eth42\niface eth42 inet dhcp\n")
 
+	c.Check(s.sctlargs, DeepEquals, []string{"restart", "networking", "--no-block"})
+}
+
+func (s *FirstBootTestSuite) TestEnableFirstEtherSomeEthFailsIfSystemctlFails(c *C) {
+	dir := c.MkDir()
+	_, err := os.Create(filepath.Join(dir, "eth42"))
+	c.Assert(err, IsNil)
+	s.sctlerr = fmt.Errorf("Error")
+
+	globs = []string{filepath.Join(dir, "eth*")}
+	c.Check(enableFirstEther(), Equals, s.sctlerr)
+	fs, _ := filepath.Glob(filepath.Join(ethdir, "*"))
+	c.Check(fs, HasLen, 1)
+	c.Check(s.sctlargs, DeepEquals, []string{"restart", "networking", "--no-block"})
 }
 
 func (s *FirstBootTestSuite) TestEnableFirstEtherBadEthDir(c *C) {

=== modified file 'systemd/systemd.go'
--- systemd/systemd.go	2015-07-23 13:15:56 +0000
+++ systemd/systemd.go	2015-07-29 09:50:51 +0000
@@ -84,6 +84,7 @@
 	StopTimeout time.Duration
 	AaProfile   string
 	IsFramework bool
+	IsNetworked bool
 	BusName     string
 	UdevAppName string
 }
@@ -205,7 +206,9 @@
 {{if .IsFramework}}Before=ubuntu-snappy.frameworks.target
 After=ubuntu-snappy.frameworks-pre.target
 Requires=ubuntu-snappy.frameworks-pre.target{{else}}After=ubuntu-snappy.frameworks.target
-Requires=ubuntu-snappy.frameworks.target{{end}}
+Requires=ubuntu-snappy.frameworks.target{{end}}{{if .IsNetworked}}
+After=snappy-wait4network.service
+Requires=snappy-wait4network.service{{end}}
 X-Snappy=yes
 
 [Service]

=== modified file 'systemd/systemd_test.go'
--- systemd/systemd_test.go	2015-07-23 13:15:56 +0000
+++ systemd/systemd_test.go	2015-07-29 09:50:51 +0000
@@ -181,6 +181,10 @@
 	expectedAppService  = fmt.Sprintf(expectedServiceFmt, "After=ubuntu-snappy.frameworks.target\nRequires=ubuntu-snappy.frameworks.target", ".mvo", "mvo", "\n", helpers.UbuntuArchitecture())
 	expectedFmkService  = fmt.Sprintf(expectedServiceFmt, "Before=ubuntu-snappy.frameworks.target\nAfter=ubuntu-snappy.frameworks-pre.target\nRequires=ubuntu-snappy.frameworks-pre.target", "", "", "\n", helpers.UbuntuArchitecture())
 	expectedDbusService = fmt.Sprintf(expectedServiceFmt, "After=ubuntu-snappy.frameworks.target\nRequires=ubuntu-snappy.frameworks.target", ".mvo", "mvo", "BusName=foo.bar.baz\nType=dbus", helpers.UbuntuArchitecture())
+
+	// things that need network
+	expectedNetAppService = fmt.Sprintf(expectedServiceFmt, "After=ubuntu-snappy.frameworks.target\nRequires=ubuntu-snappy.frameworks.target\nAfter=snappy-wait4network.service\nRequires=snappy-wait4network.service", ".mvo", "mvo", "\n", helpers.UbuntuArchitecture())
+	expectedNetFmkService = fmt.Sprintf(expectedServiceFmt, "Before=ubuntu-snappy.frameworks.target\nAfter=ubuntu-snappy.frameworks-pre.target\nRequires=ubuntu-snappy.frameworks-pre.target\nAfter=snappy-wait4network.service\nRequires=snappy-wait4network.service", "", "", "\n", helpers.UbuntuArchitecture())
 )
 
 func (s *SystemdTestSuite) TestGenAppServiceFile(c *C) {
@@ -202,6 +206,26 @@
 	c.Check(New("", nil).GenServiceFile(desc), Equals, expectedAppService)
 }
 
+func (s *SystemdTestSuite) TestGenNetAppServiceFile(c *C) {
+
+	desc := &ServiceDescription{
+		AppName:     "app",
+		ServiceName: "service",
+		Version:     "1.0",
+		Description: "descr",
+		AppPath:     "/apps/app.mvo/1.0/",
+		Start:       "bin/start",
+		Stop:        "bin/stop",
+		PostStop:    "bin/stop --post",
+		StopTimeout: time.Duration(10 * time.Second),
+		AaProfile:   "aa-profile",
+		IsNetworked: true,
+		UdevAppName: "app.mvo",
+	}
+
+	c.Check(New("", nil).GenServiceFile(desc), Equals, expectedNetAppService)
+}
+
 func (s *SystemdTestSuite) TestGenFmkServiceFile(c *C) {
 
 	desc := &ServiceDescription{
@@ -222,6 +246,27 @@
 	c.Check(New("", nil).GenServiceFile(desc), Equals, expectedFmkService)
 }
 
+func (s *SystemdTestSuite) TestGenNetFmkServiceFile(c *C) {
+
+	desc := &ServiceDescription{
+		AppName:     "app",
+		ServiceName: "service",
+		Version:     "1.0",
+		Description: "descr",
+		AppPath:     "/apps/app/1.0/",
+		Start:       "bin/start",
+		Stop:        "bin/stop",
+		PostStop:    "bin/stop --post",
+		StopTimeout: time.Duration(10 * time.Second),
+		AaProfile:   "aa-profile",
+		IsNetworked: true,
+		IsFramework: true,
+		UdevAppName: "app",
+	}
+
+	c.Check(New("", nil).GenServiceFile(desc), Equals, expectedNetFmkService)
+}
+
 func (s *SystemdTestSuite) TestGenServiceFileWithBusName(c *C) {
 
 	desc := &ServiceDescription{

=== added directory 'wait4network'
=== added file 'wait4network/wait4network.go'
--- wait4network/wait4network.go	1970-01-01 00:00:00 +0000
+++ wait4network/wait4network.go	2015-07-29 09:50:51 +0000
@@ -0,0 +1,77 @@
+// -*- Mode: Go; indent-tabs-mode: t -*-
+
+/*
+ * Copyright (C) 2014-2015 Canonical Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package main
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"time"
+)
+
+var (
+	errUsage = fmt.Errorf("Usage: %s URL MD5 TIMEOUT INTERVAL; for example: http://start.ubuntu.com/connectivity-check.html 4589f42e1546aa47ca181e5d949d310b 1s 5s", os.Args[0])
+)
+
+func die(err error) {
+	fmt.Fprintln(os.Stderr, "error:", err)
+	os.Exit(1)
+}
+
+func main() {
+	if len(os.Args) != 5 {
+		die(errUsage)
+	}
+
+	targetURL := os.Args[1]
+	targetMD5 := os.Args[2]
+	targetTimeout, err := time.ParseDuration(os.Args[3])
+	if err != nil {
+		die(err)
+	}
+	interval, err := time.ParseDuration(os.Args[4])
+	if err != nil {
+		die(err)
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Dial: func(network, addr string) (net.Conn, error) {
+				return net.DialTimeout(network, addr, targetTimeout)
+			},
+		},
+	}
+
+	ticker := time.NewTicker(interval)
+
+	for {
+		if resp, err := client.Get(targetURL); err == nil {
+			h := md5.New()
+			io.Copy(h, resp.Body)
+			if fmt.Sprintf("%x", h.Sum(nil)) == targetMD5 {
+				os.Exit(0)
+			}
+		}
+		<-ticker.C
+	}
+}