Merge lp:~pedronis/ubuntu-push/server-lego-preps into lp:ubuntu-push

[Samuele Pedroni]

teach config how to behave with anonymous fields, unexported fields, and to compose a config reading from many files; move some shared/sharable things into testing/

* teach config how to behave with anonymous fields, unexported fields, and to compose a config reading from many files

* move some shared/sharable things into testing/

[John Lenton (chipaca)]

review: Approve

=== modified file 'config/config.go'
--- config/config.go	2014-01-14 15:35:20 +0000
+++ config/config.go	2014-01-22 09:46:32 +0000
@@ -24,43 +24,69 @@
 	"io"
 	"io/ioutil"
 	"net"
+	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
 	"time"
 )
 
-// ReadConfig reads a JSON configuration into destConfig which should
-// be a pointer to a structure, it does some more configuration
-// specific error checking than plain JSON decoding and mentions
-// fields in errors . Configuration fields are expected to start with
-// lower case in the JSON object.
-func ReadConfig(r io.Reader, destConfig interface{}) error {
+func checkDestConfig(destConfig interface{}) (reflect.Value, error) {
 	destValue := reflect.ValueOf(destConfig)
 	if destValue.Kind() != reflect.Ptr || destValue.Elem().Kind() != reflect.Struct {
-		return errors.New("destConfig not *struct")
-	}
-	// do the parsing in two phases for better error handling
-	var p1 map[string]json.RawMessage
-	err := json.NewDecoder(r).Decode(&p1)
-	if err != nil {
-		return err
-	}
+		return reflect.Value{}, errors.New("destConfig not *struct")
+	}
+	return destValue, nil
+}
+
+type destField struct {
+	fld  reflect.StructField
+	dest interface{}
+}
+
+func traverseStruct(destStruct reflect.Value) <-chan destField {
+	ch := make(chan destField)
+	var traverse func(reflect.Value, chan<- destField)
+	traverse = func(destStruct reflect.Value, ch chan<- destField) {
+		structType := destStruct.Type()
+		n := structType.NumField()
+		for i := 0; i < n; i++ {
+			fld := structType.Field(i)
+			val := destStruct.Field(i)
+			if fld.PkgPath != "" { // unexported
+				continue
+			}
+			if fld.Anonymous {
+				traverse(val, ch)
+				continue
+			}
+			ch <- destField{
+				fld:  fld,
+				dest: val.Addr().Interface(),
+			}
+		}
+	}
+	go func() {
+		traverse(destStruct, ch)
+		close(ch)
+	}()
+	return ch
+}
+
+func fillDestConfig(destValue reflect.Value, p map[string]json.RawMessage) error {
 	destStruct := destValue.Elem()
-	structType := destStruct.Type()
-	n := structType.NumField()
-	for i := 0; i < n; i++ {
-		fld := structType.Field(i)
+	for destField := range traverseStruct(destStruct) {
+		fld := destField.fld
 		configName := strings.Split(fld.Tag.Get("json"), ",")[0]
 		if configName == "" {
 			configName = strings.ToLower(fld.Name[:1]) + fld.Name[1:]
 		}
-		raw, found := p1[configName]
+		raw, found := p[configName]
 		if !found { // assume all fields are mandatory for now
 			return fmt.Errorf("missing %s", configName)
 		}
-		dest := destStruct.Field(i).Addr().Interface()
-		err = json.Unmarshal([]byte(raw), dest)
+		dest := destField.dest
+		err := json.Unmarshal([]byte(raw), dest)
 		if err != nil {
 			return fmt.Errorf("%s: %v", configName, err)
 		}
@@ -68,6 +94,25 @@
 	return nil
 }
 
+// ReadConfig reads a JSON configuration into destConfig which should
+// be a pointer to a structure, it does some more configuration
+// specific error checking than plain JSON decoding and mentions
+// fields in errors . Configuration fields are expected to start with
+// lower case in the JSON object.
+func ReadConfig(r io.Reader, destConfig interface{}) error {
+	destValue, err := checkDestConfig(destConfig)
+	if err != nil {
+		return err
+	}
+	// do the parsing in two phases for better error handling
+	var p1 map[string]json.RawMessage
+	err = json.NewDecoder(r).Decode(&p1)
+	if err != nil {
+		return err
+	}
+	return fillDestConfig(destValue, p1)
+}
+
 // ConfigTimeDuration can hold a time.Duration in a configuration struct,
 // that is parsed from a string as supported by time.ParseDuration.
 type ConfigTimeDuration struct {
@@ -147,3 +192,32 @@
 	}
 	return ioutil.ReadFile(p)
 }
+
+// ReadFiles reads configuration from a set of files. Uses ReadConfig internally.
+func ReadFiles(destConfig interface{}, cfgFpaths ...string) error {
+	destValue, err := checkDestConfig(destConfig)
+	if err != nil {
+		return err
+	}
+	// do the parsing in two phases for better error handling
+	var p1 map[string]json.RawMessage
+	readOne := false
+	for _, cfgPath := range cfgFpaths {
+		if _, err := os.Stat(cfgPath); err == nil {
+			r, err := os.Open(cfgPath)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+			err = json.NewDecoder(r).Decode(&p1)
+			if err != nil {
+				return err
+			}
+			readOne = true
+		}
+	}
+	if !readOne {
+		return fmt.Errorf("no config to read")
+	}
+	return fillDestConfig(destValue, p1)
+}

=== modified file 'config/config_test.go'
--- config/config_test.go	2014-01-14 15:35:20 +0000
+++ config/config_test.go	2014-01-22 09:46:32 +0000
@@ -22,6 +22,7 @@
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 	"time"
 )
@@ -133,3 +134,57 @@
 	c.Check(err, IsNil)
 	c.Check(string(d), Equals, "Example")
 }
+
+func (s *configSuite) TestReadFiles(c *C) {
+	tmpDir := c.MkDir()
+	cfg1Path := filepath.Join(tmpDir, "cfg1.json")
+	err := ioutil.WriteFile(cfg1Path, []byte(`{"a": 42}`), os.ModePerm)
+	c.Assert(err, IsNil)
+	cfg2Path := filepath.Join(tmpDir, "cfg2.json")
+	err = ioutil.WriteFile(cfg2Path, []byte(`{"b": "x", "c_list": ["y", "z"]}`), os.ModePerm)
+	c.Assert(err, IsNil)
+	var cfg testConfig1
+	err = ReadFiles(&cfg, cfg1Path, cfg2Path)
+	c.Assert(err, IsNil)
+	c.Check(cfg.A, Equals, 42)
+	c.Check(cfg.B, Equals, "x")
+	c.Check(cfg.C, DeepEquals, []string{"y", "z"})
+}
+
+func (s *configSuite) TestReadFilesErrors(c *C) {
+	var cfg testConfig1
+	err := ReadFiles(1)
+	c.Check(err, ErrorMatches, `destConfig not \*struct`)
+	err = ReadFiles(&cfg, "non-existent")
+	c.Check(err, ErrorMatches, "no config to read")
+	err = ReadFiles(&cfg, "/root")
+	c.Check(err, ErrorMatches, ".*permission denied")
+	tmpDir := c.MkDir()
+	err = ReadFiles(&cfg, tmpDir)
+	c.Check(err, ErrorMatches, ".*is a directory")
+	brokenCfgPath := filepath.Join(tmpDir, "b.json")
+	err = ioutil.WriteFile(brokenCfgPath, []byte(`{"a"-`), os.ModePerm)
+	c.Assert(err, IsNil)
+	err = ReadFiles(&cfg, brokenCfgPath)
+	c.Check(err, NotNil)
+}
+
+type B struct {
+	BFld int
+}
+
+type A struct {
+	AFld int
+	B
+	private int
+}
+
+func (s *configSuite) TestTraverseStruct(c *C) {
+	var a A
+	var i = 1
+	for destField := range traverseStruct(reflect.ValueOf(&a).Elem()) {
+		*(destField.dest.(*int)) = i
+		i++
+	}
+	c.Check(a, DeepEquals, A{1, B{2}, 0})
+}

=== modified file 'server/acceptance/acceptance_test.go'
--- server/acceptance/acceptance_test.go	2014-01-17 18:29:24 +0000
+++ server/acceptance/acceptance_test.go	2014-01-22 09:46:32 +0000
@@ -26,6 +26,7 @@
 	. "launchpad.net/gocheck"
 	"launchpad.net/ubuntu-push/protocol"
 	"launchpad.net/ubuntu-push/server/api"
+	helpers "launchpad.net/ubuntu-push/testing"
 	"net"
 	"net/http"
 	"os"
@@ -52,17 +53,6 @@
 
 var serverCmd = flag.String("server", "", "server to test")
 
-// SourceRelative produces a path relative to the source code, makes
-// sense only for tests when the code is available on disk.
-// xxx later move it to a testing helpers package
-func SourceRelative(relativePath string) string {
-	_, file, _, ok := runtime.Caller(1)
-	if !ok {
-		panic("failed to get source filename using Caller()")
-	}
-	return filepath.Join(filepath.Dir(file), relativePath)
-}
-
 func testServerConfig(addr, httpAddr string) map[string]interface{} {
 	cfg := map[string]interface{}{
 		"exchange_timeout":   "0.1s",
@@ -70,8 +60,8 @@
 		"session_queue_size": 10,
 		"broker_queue_size":  100,
 		"addr":               addr,
-		"key_pem_file":       SourceRelative("config/testing.key"),
-		"cert_pem_file":      SourceRelative("config/testing.cert"),
+		"key_pem_file":       helpers.SourceRelative("config/testing.key"),
+		"cert_pem_file":      helpers.SourceRelative("config/testing.cert"),
 		"http_addr":          httpAddr,
 		"http_read_timeout":  "1s",
 		"http_write_timeout": "1s",
@@ -80,7 +70,7 @@
 }
 
 func testClientSession(addr string, deviceId string, reportPings bool) *ClientSession {
-	certPEMBlock, err := ioutil.ReadFile(SourceRelative("config/testing.cert"))
+	certPEMBlock, err := ioutil.ReadFile(helpers.SourceRelative("config/testing.cert"))
 	if err != nil {
 		panic(fmt.Sprintf("could not read config/testing.cert: %v", err))
 	}

=== modified file 'server/listener/listener.go'
--- server/listener/listener.go	2014-01-16 13:43:00 +0000
+++ server/listener/listener.go	2014-01-22 09:46:32 +0000
@@ -50,6 +50,9 @@
 		SessionTicketsDisabled: true,
 	}
 	lst, err := tls.Listen("tcp", cfg.Addr(), tlsCfg)
+	if err != nil {
+		return nil, err
+	}
 	return &DeviceListener{lst}, err
 }
 

=== modified file 'server/listener/listener_test.go'
--- server/listener/listener_test.go	2014-01-16 13:43:00 +0000
+++ server/listener/listener_test.go	2014-01-22 09:46:32 +0000
@@ -58,31 +58,12 @@
 	return cfg.addr
 }
 
-// key&cert generated with go run /usr/lib/go/src/pkg/crypto/tls/generate_cert.go -ca -host localhost -rsa-bits 512 -duration 87600h
-
 func (cfg *testDevListenerCfg) KeyPEMBlock() []byte {
-	return []byte(`-----BEGIN RSA PRIVATE KEY-----
-MIIBPAIBAAJBAPw+niki17X2qALE2A2AzE1q5dvK9CI4OduRtT9IgbFLC6psqAT2
-1NA+QbY17nWSSpyP65zkMkwKXrbDzstwLPkCAwEAAQJAKwXbIBULScP6QA6m8xam
-wgWbkvN41GVWqPafPV32kPBvKwSc+M1e+JR7g3/xPZE7TCELcfYi4yXEHZZI3Pbh
-oQIhAP/UsgJbsfH1GFv8Y8qGl5l/kmwwkwHhuKvEC87Yur9FAiEA/GlQv3ZfaXnT
-lcCFT0aL02O0RDiRYyMUG/JAZQJs6CUCIQCHO5SZYIUwxIGK5mCNxxXOAzyQSiD7
-hqkKywf+4FvfDQIhALa0TLyqJFom0t7c4iIGAIRc8UlIYQSPiajI64+x9775AiEA
-0v4fgSK/Rq059zW1753JjuB6aR0Uh+3RqJII4dUR1Wg=
------END RSA PRIVATE KEY-----`)
+	return helpers.TestKeyPEMBlock
 }
 
 func (cfg *testDevListenerCfg) CertPEMBlock() []byte {
-	return []byte(`-----BEGIN CERTIFICATE-----
-MIIBYzCCAQ+gAwIBAgIBADALBgkqhkiG9w0BAQUwEjEQMA4GA1UEChMHQWNtZSBD
-bzAeFw0xMzEyMTkyMDU1NDNaFw0yMzEyMTcyMDU1NDNaMBIxEDAOBgNVBAoTB0Fj
-bWUgQ28wWjALBgkqhkiG9w0BAQEDSwAwSAJBAPw+niki17X2qALE2A2AzE1q5dvK
-9CI4OduRtT9IgbFLC6psqAT21NA+QbY17nWSSpyP65zkMkwKXrbDzstwLPkCAwEA
-AaNUMFIwDgYDVR0PAQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud
-EwEB/wQFMAMBAf8wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAsGCSqGSIb3
-DQEBBQNBAFqiVI+Km2XPSO+pxITaPvhmuzg+XG3l1+2di3gL+HlDobocjBqRctRU
-YySO32W07acjGJmCHUKpCJuq9X8hpmk=
------END CERTIFICATE-----`)
+	return helpers.TestCertPEMBlock
 }
 
 func (s *listenerSuite) TestDeviceListen(c *C) {

=== modified file 'testing/helpers.go'
--- testing/helpers.go	2014-01-14 15:35:20 +0000
+++ testing/helpers.go	2014-01-22 09:46:32 +0000
@@ -19,6 +19,8 @@
 
 import (
 	"bytes"
+	"path/filepath"
+	"runtime"
 	"sync"
 )
 
@@ -45,3 +47,13 @@
 	defer buf.lock.Unlock()
 	return buf.Buffer.String()
 }
+
+// SourceRelative produces a path relative to the source code, makes
+// sense only for tests when the code is available on disk.
+func SourceRelative(relativePath string) string {
+	_, file, _, ok := runtime.Caller(1)
+	if !ok {
+		panic("failed to get source filename using Caller()")
+	}
+	return filepath.Join(filepath.Dir(file), relativePath)
+}

=== added file 'testing/tls.go'
--- testing/tls.go	1970-01-01 00:00:00 +0000
+++ testing/tls.go	2014-01-22 09:46:32 +0000
@@ -0,0 +1,41 @@
+/*
+ Copyright 2013-2014 Canonical Ltd.
+
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License version 3, as published
+ by the Free Software Foundation.
+
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranties of
+ MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+ PURPOSE.  See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package testing
+
+// key&cert generated with go run /usr/lib/go/src/pkg/crypto/tls/generate_cert.go -ca -host localhost -rsa-bits 512 -duration 87600h
+var (
+	TestKeyPEMBlock = []byte(`-----BEGIN RSA PRIVATE KEY-----
+MIIBPAIBAAJBAPw+niki17X2qALE2A2AzE1q5dvK9CI4OduRtT9IgbFLC6psqAT2
+1NA+QbY17nWSSpyP65zkMkwKXrbDzstwLPkCAwEAAQJAKwXbIBULScP6QA6m8xam
+wgWbkvN41GVWqPafPV32kPBvKwSc+M1e+JR7g3/xPZE7TCELcfYi4yXEHZZI3Pbh
+oQIhAP/UsgJbsfH1GFv8Y8qGl5l/kmwwkwHhuKvEC87Yur9FAiEA/GlQv3ZfaXnT
+lcCFT0aL02O0RDiRYyMUG/JAZQJs6CUCIQCHO5SZYIUwxIGK5mCNxxXOAzyQSiD7
+hqkKywf+4FvfDQIhALa0TLyqJFom0t7c4iIGAIRc8UlIYQSPiajI64+x9775AiEA
+0v4fgSK/Rq059zW1753JjuB6aR0Uh+3RqJII4dUR1Wg=
+-----END RSA PRIVATE KEY-----`)
+
+	TestCertPEMBlock = []byte(`-----BEGIN CERTIFICATE-----
+MIIBYzCCAQ+gAwIBAgIBADALBgkqhkiG9w0BAQUwEjEQMA4GA1UEChMHQWNtZSBD
+bzAeFw0xMzEyMTkyMDU1NDNaFw0yMzEyMTcyMDU1NDNaMBIxEDAOBgNVBAoTB0Fj
+bWUgQ28wWjALBgkqhkiG9w0BAQEDSwAwSAJBAPw+niki17X2qALE2A2AzE1q5dvK
+9CI4OduRtT9IgbFLC6psqAT21NA+QbY17nWSSpyP65zkMkwKXrbDzstwLPkCAwEA
+AaNUMFIwDgYDVR0PAQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud
+EwEB/wQFMAMBAf8wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAsGCSqGSIb3
+DQEBBQNBAFqiVI+Km2XPSO+pxITaPvhmuzg+XG3l1+2di3gL+HlDobocjBqRctRU
+YySO32W07acjGJmCHUKpCJuq9X8hpmk=
+-----END CERTIFICATE-----`)
+)