Merge lp:~jamesodhunt/snappy/install.yaml into lp:~snappy-dev/snappy/snappy-moved-to-github

[James Hunt]

* snappy/systemimage_test.go: Removed original sideload tests.
* snappy/provisioning_test.go: Added new sideload tests.

Change the sideload-detection strategy by having snappy parse an install.yaml file generated by the provisioning tool (ubuntu-device-flash).

This MP should be considered in combination with:

https://code.launchpad.net/~jamesodhunt/goget-ubuntu-touch/record-sideload/+merge/256190

[Sergio Schvezov (sergiusens)]

it looks generally good, but I would prefer to have this in it's own package if possible

review: Needs Fixing

[Sergio Schvezov (sergiusens)]

A couple more comments and questions

[James Hunt (jamesodhunt)]

Thanks for reviewing. Branch updated.

[James Hunt (jamesodhunt)]

...aaaaand updated again.

[Sergio Schvezov (sergiusens)]

a couple of comments

[lp:~jamesodhunt/snappy/install.yaml]

393. By James Hunt on 2015-04-21

* Sync with lp:snappy.

394. By James Hunt on 2015-04-23

* Change InstallYamlFile to a filename as the path is going to have to
    bootloader-specific (due to grub's bind-mount).
* IsSideLoaded() (was SideLoadedSystem()) now takes a boot directory
  argument.

395. By James Hunt on 2015-04-23

* Sync with lp:snappy.

396. By James Hunt on 2015-04-23

* Comments.

397. By James Hunt on 2015-04-24

* provisioning/provisioning.go: Comments.
* provisioning/provisioning_test.go: Removed Version values to be
  consistent with soon-to-be-current ubutu-device-flash(1) behaviour.

398. By James Hunt on 2015-04-24

* Sync with lp:snappy.

[James Hunt (jamesodhunt)]

Hi Sergio - branch updated. Note that I haven't made the change to parseInstallYamlData() since that code was written specifically to be consistent with parsePackageYamlData().

Maybe we can deal with further refactoring on a separate MP ? :)

[Michael Vogt (mvo)]

Just one drive by comment, looks good otherwise even though I am not happy that partition needs that interface addition. I guess its hard to solve without as there is no other place that the file could be put that is safe from a factory reset(?).

review: Needs Information

[lp:~jamesodhunt/snappy/install.yaml]

399. By James Hunt on 2015-05-01

* Sync with lp:snappy.

400. By James Hunt on 2015-05-01

* snappy/systemimage_test.go: Added back TestCannotUpdateIfSideLoaded()
  to ensure Install() and Update() fail if the system is sideloaded.

[James Hunt (jamesodhunt)]

Thanks - branch updated.

> as there is no other place that the file could be put that is safe from a factory reset(?).
That's right.

Unmerged revisions

=== modified file 'partition/bootloader.go'
--- partition/bootloader.go	2015-04-14 20:37:54 +0000
+++ partition/bootloader.go	2015-05-01 09:42:40 +0000
@@ -80,6 +80,10 @@
 
 	// Return the additional required chroot bind mounts for this bootloader
 	AdditionalBindMounts() []string
+
+	// BootDir returns the (writable) bootloader-specific boot
+	// directory.
+	BootDir() string
 }
 
 type bootloaderType struct {

=== modified file 'partition/bootloader_grub.go'
--- partition/bootloader_grub.go	2015-04-30 11:28:39 +0000
+++ partition/bootloader_grub.go	2015-05-01 09:42:40 +0000
@@ -164,3 +164,7 @@
 	// well known location for its configuration
 	return []string{"/boot/grub"}
 }
+
+func (g *grub) BootDir() string {
+	return bootloaderGrubDir
+}

=== modified file 'partition/bootloader_uboot.go'
--- partition/bootloader_uboot.go	2015-04-30 11:28:39 +0000
+++ partition/bootloader_uboot.go	2015-05-01 09:42:40 +0000
@@ -395,3 +395,7 @@
 	// nothing additional to system-boot required on uboot
 	return []string{}
 }
+
+func (u *uboot) BootDir() string {
+	return bootloaderUbootDir
+}

=== modified file 'partition/partition.go'
--- partition/partition.go	2015-03-30 14:56:46 +0000
+++ partition/partition.go	2015-05-01 09:42:40 +0000
@@ -124,6 +124,10 @@
 
 	// run the function f with the otherRoot mounted
 	RunWithOther(rw MountOption, f func(otherRoot string) (err error)) (err error)
+
+	// Returns the full path to the (mounted and writable)
+	// bootloader-specific boot directory.
+	GetBootloaderDir() string
 }
 
 // mountEntry represents a mount this package has created.
@@ -821,3 +825,14 @@
 
 	return bootloader.HandleAssets()
 }
+
+// GetBootloaderDir returns the full path to the (mounted and writable)
+// bootloader-specific boot directory.
+func (p *Partition) GetBootloaderDir() string {
+	bootloader, err := getBootloader(p)
+	if err != nil {
+		return ""
+	}
+
+	return bootloader.BootDir()
+}

=== modified file 'partition/partition_test.go'
--- partition/partition_test.go	2015-03-30 14:42:31 +0000
+++ partition/partition_test.go	2015-05-01 09:42:40 +0000
@@ -451,6 +451,10 @@
 	return nil
 }
 
+func (b *mockBootloader) BootDir() string {
+	return ""
+}
+
 func (s *PartitionTestSuite) TestToggleBootloaderRootfs(c *C) {
 	runCommand = mockRunCommand
 	b := &mockBootloader{}

=== added directory 'provisioning'
=== added file 'provisioning/provisioning.go'
--- provisioning/provisioning.go	1970-01-01 00:00:00 +0000
+++ provisioning/provisioning.go	2015-05-01 09:42:40 +0000
@@ -0,0 +1,127 @@
+/*
+ * Copyright (C) 2014-2015 Canonical Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package provisioning
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+	"time"
+
+	"launchpad.net/snappy/helpers"
+	"launchpad.net/snappy/logger"
+
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	// InstallYamlFile is the name of the file created by
+	// ubuntu-device-flash(1), created at system installation time,
+	// that contains metadata on the installation.
+	//
+	// XXX: Public for ubuntu-device-flash(1)
+	InstallYamlFile = "install.yaml"
+
+	// ErrNoInstallYaml is emitted when InstallYamlFile does not exist.
+	ErrNoInstallYaml = fmt.Errorf("no %s", InstallYamlFile)
+)
+
+// InstallMeta encapsulates the metadata for a system install.
+type InstallMeta struct {
+	Timestamp         time.Time
+	InitialVersion    string `yaml:"initial-version"`
+	SystemImageServer string `yaml:"system-image-server"`
+}
+
+// InstallTool encapsulates metadata on the tool used to create the
+// system image.
+type InstallTool struct {
+	Name    string
+	Path    string
+	Version string
+}
+
+// InstallOptions summarises the options used when creating the system image.
+type InstallOptions struct {
+	Size          int64  `yaml:"size"`
+	SizeUnit      string `yaml:"size-unit"`
+	Output        string
+	Channel       string
+	DevicePart    string `yaml:"device-part,omitempty"`
+	Oem           string
+	DeveloperMode bool `yaml:"developer-mode,omitempty"`
+}
+
+// InstallYaml represents 'InstallYamlFile'
+//
+// XXX: Public for ubuntu-device-flash
+type InstallYaml struct {
+	InstallMeta    `yaml:"meta"`
+	InstallTool    `yaml:"tool"`
+	InstallOptions `yaml:"options"`
+}
+
+func parseInstallYaml(path string) (*InstallYaml, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, ErrNoInstallYaml
+	}
+
+	return parseInstallYamlData(data)
+}
+
+func parseInstallYamlData(yamlData []byte) (*InstallYaml, error) {
+	var i InstallYaml
+	err := yaml.Unmarshal(yamlData, &i)
+	if err != nil {
+		log.Printf("Cannot parse %q", yamlData)
+		return nil, err
+	}
+
+	return &i, nil
+}
+
+// IsSideLoaded determines if the system was installed using a
+// custom enablement part.
+func IsSideLoaded(bootloaderDir string) bool {
+	file := filepath.Join(bootloaderDir, InstallYamlFile)
+
+	if !helpers.FileExists(file) {
+		// the system may have been sideloaded, but we have no
+		// way of knowing :-(
+		return false
+	}
+
+	InstallYaml, err := parseInstallYaml(file)
+	if err != nil {
+		logger.LogError(err)
+		// file isn't parseable, so let's assume system is sideloaded
+		return true
+	}
+
+	if InstallYaml.InstallOptions.DevicePart != "" {
+		// system was created with something like:
+		//
+		//  "ubuntu-device-flash [...] --device-part=unofficial-assets.tar.xz ..."
+		//
+		return true
+	}
+
+	return false
+}

=== added file 'provisioning/provisioning_test.go'
--- provisioning/provisioning_test.go	1970-01-01 00:00:00 +0000
+++ provisioning/provisioning_test.go	2015-05-01 09:42:40 +0000
@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2014-2015 Canonical Ltd
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 3 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package provisioning
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner
+func Test(t *testing.T) { TestingT(t) }
+
+type ProvisioningTestSuite struct {
+	tempdir         string
+	installYamlFile string
+}
+
+var _ = Suite(&ProvisioningTestSuite{})
+
+var yamlData = `
+meta:
+  timestamp: 2015-04-20T14:15:39.013515821+01:00
+  initial-revision: r345
+  system-image-server: http://system-image.ubuntu.com
+
+tool:
+  name: ubuntu-device-flash
+  path: /usr/bin/ubuntu-device-flash
+  version: ""
+
+options:
+  size: 3
+  size-unit: GB
+  output: /tmp/bbb.img
+  channel: ubuntu-core/devel-proposed
+  device-part: /some/path/file.tgz
+  developer-mode: true
+`
+
+var yamlDataNoDevicePart = `
+meta:
+  timestamp: 2015-04-20T14:15:39.013515821+01:00
+  initial-revision: r345
+  system-image-server: http://system-image.ubuntu.com
+
+tool:
+  name: ubuntu-device-flash
+  path: /usr/bin/ubuntu-device-flash
+  version: ""
+
+options:
+  size: 3
+  size-unit: GB
+  output: /tmp/bbb.img
+  channel: ubuntu-core/devel-proposed
+  developer-mode: true
+`
+
+var garbageData = `Fooled you!?`
+
+func (ts *ProvisioningTestSuite) SetUpTest(c *C) {
+	ts.tempdir = c.MkDir()
+
+	ts.installYamlFile = filepath.Join(ts.tempdir, "install.yaml")
+
+	InstallYamlFile = ts.installYamlFile
+	os.Remove(InstallYamlFile)
+}
+
+func (ts *ProvisioningTestSuite) TestSideLoadedSystemNoInstallYaml(c *C) {
+	c.Assert(IsSideLoaded(""), Equals, false)
+}
+
+func (ts *ProvisioningTestSuite) TestSideLoadedSystem(c *C) {
+	c.Assert(IsSideLoaded(""), Equals, false)
+
+	err := ioutil.WriteFile(InstallYamlFile, []byte(yamlData), 0750)
+	c.Assert(err, IsNil)
+
+	c.Assert(IsSideLoaded(""), Equals, true)
+
+	os.Remove(InstallYamlFile)
+	c.Assert(IsSideLoaded(""), Equals, false)
+}
+
+func (ts *ProvisioningTestSuite) TestSideLoadedSystemNoDevicePart(c *C) {
+
+	c.Assert(IsSideLoaded(""), Equals, false)
+
+	err := ioutil.WriteFile(InstallYamlFile, []byte(yamlDataNoDevicePart), 0750)
+	c.Assert(err, IsNil)
+
+	c.Assert(IsSideLoaded(""), Equals, false)
+
+	os.Remove(InstallYamlFile)
+	c.Assert(IsSideLoaded(""), Equals, false)
+}
+
+func (ts *ProvisioningTestSuite) TestSideLoadedSystemGarbageInstallYaml(c *C) {
+	c.Assert(IsSideLoaded(""), Equals, false)
+
+	err := ioutil.WriteFile(InstallYamlFile, []byte(garbageData), 0750)
+	c.Assert(err, IsNil)
+
+	// we assume sideloaded if the file isn't parseable
+	c.Assert(IsSideLoaded(""), Equals, true)
+
+	os.Remove(InstallYamlFile)
+	c.Assert(IsSideLoaded(""), Equals, false)
+}
+
+func (ts *ProvisioningTestSuite) TestParseInstallYaml(c *C) {
+
+	_, err := parseInstallYaml(InstallYamlFile)
+	c.Check(err, Equals, ErrNoInstallYaml)
+
+	err = ioutil.WriteFile(InstallYamlFile, []byte(yamlData), 0750)
+	c.Check(err, IsNil)
+	_, err = parseInstallYaml(InstallYamlFile)
+	c.Check(err, IsNil)
+
+	err = ioutil.WriteFile(InstallYamlFile, []byte(yamlDataNoDevicePart), 0750)
+	c.Check(err, IsNil)
+	_, err = parseInstallYaml(InstallYamlFile)
+	c.Check(err, IsNil)
+
+	err = ioutil.WriteFile(InstallYamlFile, []byte(garbageData), 0750)
+	c.Check(err, IsNil)
+	_, err = parseInstallYaml(InstallYamlFile)
+	c.Check(err, Not(Equals), nil)
+}
+
+func (ts *ProvisioningTestSuite) TestParseInstallYamlData(c *C) {
+
+	_, err := parseInstallYamlData([]byte(""))
+	c.Check(err, IsNil)
+
+	_, err = parseInstallYamlData([]byte(yamlData))
+	c.Check(err, IsNil)
+
+	_, err = parseInstallYamlData([]byte(yamlDataNoDevicePart))
+	c.Check(err, IsNil)
+
+	_, err = parseInstallYamlData([]byte(garbageData))
+	c.Check(err, Not(Equals), nil)
+}

=== modified file 'snappy/systemimage.go'
--- snappy/systemimage.go	2015-04-15 20:39:24 +0000
+++ snappy/systemimage.go	2015-05-01 09:42:40 +0000
@@ -31,6 +31,7 @@
 	"launchpad.net/snappy/helpers"
 	"launchpad.net/snappy/partition"
 	"launchpad.net/snappy/progress"
+	"launchpad.net/snappy/provisioning"
 
 	"github.com/mvo5/goconfigparser"
 )
@@ -51,16 +52,6 @@
 	// The full path to this file needs to be passed to
 	// systemImageCli when querying a different rootfs.
 	systemImageClientConfig = "/etc/system-image/client.ini"
-
-	// Full path to file, which if present, marks the system as
-	// having been "sideloaded", in other words having been created
-	// like this:
-	//
-	// "ubuntu-device-flash --device-part=unofficial-assets.tar.xz ..."
-	//
-	// Sideloaded systems cannot be safely upgraded since there are
-	// no device-part updates on the system-image server.
-	sideLoadedMarkerFile = "/boot/.sideloaded"
 )
 
 var (
@@ -167,16 +158,9 @@
 	return s.partition.ToggleNextBoot()
 }
 
-// sideLoadedSystem determines if the system was installed using a
-// custom enablement part.
-func sideLoadedSystem() bool {
-	path := filepath.Join(systemImageRoot, sideLoadedMarkerFile)
-	return helpers.FileExists(path)
-}
-
 // Install installs the snap
 func (s *SystemImagePart) Install(pb progress.Meter, flags InstallFlags) (name string, err error) {
-	if sideLoadedSystem() {
+	if provisioning.IsSideLoaded(s.partition.GetBootloaderDir()) {
 		return "", ErrSideLoaded
 	}
 

=== modified file 'snappy/systemimage_test.go'
--- snappy/systemimage_test.go	2015-04-22 10:35:09 +0000
+++ snappy/systemimage_test.go	2015-05-01 09:42:40 +0000
@@ -27,6 +27,7 @@
 	"testing"
 
 	partition "launchpad.net/snappy/partition"
+	"launchpad.net/snappy/provisioning"
 
 	. "launchpad.net/gocheck"
 )
@@ -168,6 +169,13 @@
 	return f("/other")
 }
 
+// used by GetBootLoaderDir(), used to test sideload logic.
+var tempBootDir string
+
+func (p *MockPartition) GetBootloaderDir() string {
+	return tempBootDir
+}
+
 func (s *SITestSuite) TestSystemImagePartInstallUpdatesPartition(c *C) {
 	// add a update
 	mockSystemImageIndexJSON = fmt.Sprintf(mockSystemImageIndexJSONTemplate, "2")
@@ -359,43 +367,51 @@
 }
 
 func (s *SITestSuite) TestCannotUpdateIfSideLoaded(c *C) {
+	var err error
+	var yamlData = `
+meta:
+  timestamp: 2015-04-20T14:15:39.013515821+01:00
+  initial-revision: r345
+  system-image-server: http://system-image.ubuntu.com
+
+tool:
+  name: ubuntu-device-flash
+  path: /usr/bin/ubuntu-device-flash
+  version: ""
+
+options:
+  size: 3
+  size-unit: GB
+  output: /tmp/bbb.img
+  channel: ubuntu-core/devel-proposed
+  device-part: /some/path/file.tgz
+  developer-mode: true
+`
+	tempBootDir, err = ioutil.TempDir("", "")
+	c.Assert(err, IsNil)
+
 	parts, err := s.systemImage.Updates()
 
 	sp := parts[0].(*SystemImagePart)
 	mockPartition := MockPartition{}
 	sp.partition = &mockPartition
 
-	sideLoaded := filepath.Join(systemImageRoot, "/boot/.sideloaded")
+	bootDir := sp.partition.GetBootloaderDir()
+
+	sideLoaded := filepath.Join(bootDir, provisioning.InstallYamlFile)
 
 	err = os.MkdirAll(filepath.Dir(sideLoaded), 0775)
 	c.Assert(err, IsNil)
 
-	err = ioutil.WriteFile(sideLoaded, []byte(""), 0640)
+	err = ioutil.WriteFile(sideLoaded, []byte(yamlData), 0640)
 	c.Assert(err, IsNil)
 
 	pb := &MockProgressMeter{}
-	// do the install
+
+	// Ensure the install fails if the system is sideloaded
 	_, err = sp.Install(pb, 0)
 	c.Assert(err, Equals, ErrSideLoaded)
-}
-
-func (s *SITestSuite) TestSideLoadedSystem(c *C) {
-
-	c.Assert(sideLoadedSystem(), Equals, false)
-
-	sideLoaded := filepath.Join(systemImageRoot, sideLoadedMarkerFile)
-
-	err := os.MkdirAll(filepath.Dir(sideLoaded), 0775)
-	c.Assert(err, IsNil)
-
-	c.Assert(sideLoadedSystem(), Equals, false)
-
-	err = ioutil.WriteFile(sideLoaded, []byte(""), 0640)
-	c.Assert(err, IsNil)
-
-	c.Assert(sideLoadedSystem(), Equals, true)
-
-	os.Remove(sideLoaded)
-
-	c.Assert(sideLoadedSystem(), Equals, false)
+
+	os.Remove(tempBootDir)
+	tempBootDir = ""
 }