Merge lp:~sergiusens/goget-ubuntu-touch/reducedSize into lp:~lool/goget-ubuntu-touch/fix-ftd-typos

[Sergio Schvezov]

Unmerged revisions

141. By Sergio Schvezov on 2015-03-18

Making the total size 97.5% of the real total size for core and GB sized disks (not GiB)

140. By Sergio Schvezov on 2015-03-06

Releasing 0.18

139. By Sergio Schvezov on 2015-03-06

Populate both /boot/uboot/{a,b} when creating a ubuntu-core image.

138. By Sergio Schvezov on 2015-03-03

Releasing 0.17

137. By Sergio Schvezov on 2015-03-03

ubuntu-device-flash: more helpful error message on failed push.

136. By Sergio Schvezov on 2015-01-29

Releasing 0.16

* ubuntu-device-flash: oem part to allow for dtb overrides.
* debian/control: dep on fakeroot for ubuntu-device-flash.

135. By Sergio Schvezov on 2015-01-29

debian/control: u-d-f dependency on fakeroot

134. By Sergio Schvezov on 2015-01-29

Allowing oem part to contain a dtb to override the dtb provided by the device part for the platform

133. By Sergio Schvezov on 2015-01-29

Releasing 0.15

[ Loïc Minier ]
* ubuntu-device-flash: fixed ftdfile -> fdtfile typo.
[ Sergio Schvezov ]
* ubuntu-device-flash:
  - future proof efi support for grub and provisioning boot partition.
  - using test instead of load in snappy-commands.txt

132. By Sergio Schvezov on 2015-01-28

Future proof support for efi and boot partition support.

=== modified file 'debian/changelog'
--- debian/changelog	2015-01-21 23:25:47 +0000
+++ debian/changelog	2015-03-18 03:33:51 +0000
@@ -1,3 +1,38 @@
+goget-ubuntu-touch (0.18-0ubuntu1) vivid; urgency=medium
+
+  [ Michael Vogt ]
+  * Populate both /boot/uboot/{a,b} when creating a ubuntu-core image.
+    (LP: #1428640)
+
+ -- Sergio Schvezov <[e-mail]>  Fri, 06 Mar 2015 11:21:17 -0300
+
+goget-ubuntu-touch (0.17-0ubuntu1) vivid; urgency=medium
+
+  [ Jani Monoses ]
+  * ubuntu-device-flash: more helpful error message on failed push.
+    (LP: #1351039)
+
+ -- Sergio Schvezov <[e-mail]>  Tue, 03 Mar 2015 07:27:17 -0300
+
+goget-ubuntu-touch (0.16-0ubuntu1) vivid; urgency=medium
+
+  * ubuntu-device-flash: oem part to allow for dtb overrides.
+  * debian/control: dep on fakeroot for ubuntu-device-flash.
+
+ -- Sergio Schvezov <[e-mail]>  Thu, 29 Jan 2015 15:23:27 -0300
+
+goget-ubuntu-touch (0.15-0ubuntu1) vivid; urgency=medium
+
+  [ Loïc Minier ]
+  * ubuntu-device-flash: fixed ftdfile -> fdtfile typo.
+
+  [ Sergio Schvezov ]
+  * ubuntu-device-flash:
+    - future proof efi support for grub and provisioning boot partition.
+    - using test instead of load in snappy-commands.txt
+
+ -- Sergio Schvezov <[e-mail]>  Wed, 28 Jan 2015 19:53:33 -0300
+
 goget-ubuntu-touch (0.14-0ubuntu1) vivid; urgency=medium
 
   * Partition sizing multiplier matters for the emulator disk. (LP: #1412495)

=== modified file 'debian/control'
--- debian/control	2015-01-15 22:33:59 +0000
+++ debian/control	2015-03-18 03:33:51 +0000
@@ -19,6 +19,7 @@
 Architecture: any
 Depends: android-tools-adb,
          android-tools-fastboot,
+         fakeroot,
          kpartx,
          qemu-user-static,
          ${misc:Depends},

=== modified file 'devices/adb.go'
--- devices/adb.go	2014-09-25 13:40:20 +0000
+++ devices/adb.go	2015-03-18 03:33:51 +0000
@@ -76,7 +76,11 @@
 func (adb AndroidDebugBridge) Push(src, dst string) (err error) {
 	// TODO add file path verification
 	cmd := append(adb.params, []string{"push", src, dst}...)
-	return exec.Command(adbCommand, cmd...).Run()
+	b, err := exec.Command(adbCommand, cmd...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error pushing: %s\n", b)
+	}
+	return nil
 }
 
 // Pull copies a file from src to dst over the adb server

=== modified file 'diskimage/common.go'
--- diskimage/common.go	2015-01-15 21:05:01 +0000
+++ diskimage/common.go	2015-03-18 03:33:51 +0000
@@ -11,6 +11,7 @@
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 )
 
@@ -52,7 +53,7 @@
 type CoreImage interface {
 	Image
 	SystemImage
-	SetupBoot() error
+	SetupBoot(OemDescription) error
 	FlashExtra(string) error
 }
 
@@ -64,6 +65,18 @@
 	Bootloader      string `yaml:"bootloader"`
 }
 
+type OemDescription struct {
+	Name     string `yaml:"name"`
+	Version  string `yaml:"version"`
+	Hardware struct {
+		Dtb string `yaml:"dtb,omitempty"`
+	} `yaml:"hardware,omitempty"`
+}
+
+func (o OemDescription) InstallPath() string {
+	return filepath.Join("/oem", o.Name, o.Version)
+}
+
 func sectorSize(dev string) (string, error) {
 	out, err := exec.Command("blockdev", "--getss", dev).CombinedOutput()
 	if err != nil {

=== modified file 'diskimage/core_grub.go'
--- diskimage/core_grub.go	2015-01-21 23:22:53 +0000
+++ diskimage/core_grub.go	2015-03-18 03:33:51 +0000
@@ -247,6 +247,19 @@
 	return filepath.Join(img.baseMount, string(writableDir))
 }
 
+// Boot returns the system-boot path
+func (img CoreGrubImage) Boot() string {
+	if img.parts == nil {
+		panic("img is not setup with partitions")
+	}
+
+	if img.baseMount == "" {
+		panic("img not mounted")
+	}
+
+	return filepath.Join(img.baseMount, string(bootDir))
+}
+
 //System returns the system path
 func (img CoreGrubImage) System() string {
 	if img.parts == nil {
@@ -268,7 +281,7 @@
 	return img.baseMount
 }
 
-func (img *CoreGrubImage) SetupBoot() error {
+func (img *CoreGrubImage) SetupBoot(oem OemDescription) error {
 	for _, dev := range []string{"dev", "proc", "sys"} {
 		src := filepath.Join("/", dev)
 		dst := filepath.Join(img.System(), dev)
@@ -303,6 +316,30 @@
 	}
 	defer unmount(rootDevPath)
 
+	efiDir := filepath.Join(img.System(), "boot", "efi")
+	if err := os.MkdirAll(efiDir, 0755); err != nil {
+		return fmt.Errorf("unable to create %s dir: %s", efiDir, err)
+	}
+
+	if err := bindMount(img.Boot(), efiDir); err != nil {
+		return err
+	}
+	defer unmount(efiDir)
+
+	// create efi layout
+	efiGrubDir := filepath.Join(img.System(), "boot", "efi", "EFI", "ubuntu", "grub")
+	if err := os.MkdirAll(efiGrubDir, 0755); err != nil {
+		return fmt.Errorf("unable to create %s dir: %s", efiGrubDir, err)
+	}
+
+	bootGrubDir := filepath.Join(img.System(), "boot", "grub")
+
+	if err := bindMount(efiGrubDir, bootGrubDir); err != nil {
+		return err
+	}
+	defer unmount(bootGrubDir)
+
+	// install grub
 	if out, err := exec.Command("chroot", img.System(), "grub-install", "/root_dev").CombinedOutput(); err != nil {
 		return fmt.Errorf("unable to install grub: %s", out)
 	}

=== modified file 'diskimage/core_uboot.go'
--- diskimage/core_uboot.go	2015-01-28 13:04:27 +0000
+++ diskimage/core_uboot.go	2015-03-18 03:33:51 +0000
@@ -72,7 +72,7 @@
 snappy_mode=regular
 # if we're trying a new version, check if stamp file is already there to revert
 # to other version
-snappy_boot=if test "${snappy_mode}" = "try"; then if load mmc ${bootpart} ${loadaddr} ${snappy_stamp} 0; then if test "${snappy_ab}" = "a"; then setenv snappy_ab "b"; else setenv snappy_ab "a"; fi; else fatwrite mmc ${mmcdev}:${mmcpart} 0x0 ${snappy_stamp} 0; fi; fi; run loadfiles; setenv mmcroot /dev/disk/by-label/system-${snappy_ab} ${snappy_cmdline}; run mmcargs; bootz ${loadaddr} ${initrd_addr}:${initrd_size} ${fdtaddr}
+snappy_boot=if test "${snappy_mode}" = "try"; then if test -e mmc ${bootpart} ${snappy_stamp}; then if test "${snappy_ab}" = "a"; then setenv snappy_ab "b"; else setenv snappy_ab "a"; fi; else fatwrite mmc ${mmcdev}:${mmcpart} 0x0 ${snappy_stamp} 0; fi; fi; run loadfiles; setenv mmcroot /dev/disk/by-label/system-${snappy_ab} ${snappy_cmdline}; run mmcargs; bootz ${loadaddr} ${initrd_addr}:${initrd_size} ${fdtaddr}
 `
 
 type FlashInstructions struct {
@@ -290,12 +290,11 @@
 	return img.baseMount
 }
 
-func (img CoreUBootImage) SetupBoot() error {
+func (img CoreUBootImage) SetupBoot(oem OemDescription) error {
 	// destinations
 	bootPath := filepath.Join(img.baseMount, string(bootDir))
 	bootAPath := filepath.Join(bootPath, "a")
 	bootBPath := filepath.Join(bootPath, "b")
-	bootDtbPath := filepath.Join(bootAPath, "dtbs")
 	bootuEnvPath := filepath.Join(bootPath, "uEnv.txt")
 	bootSnappySystemPath := filepath.Join(bootPath, "snappy-system.txt")
 
@@ -304,29 +303,37 @@
 	kernelPath := filepath.Join(img.baseMount, img.hardware.Kernel)
 	initrdPath := filepath.Join(img.baseMount, img.hardware.Initrd)
 
-	// create layout
-	if err := os.MkdirAll(bootDtbPath, 0755); err != nil {
-		return err
-	}
-
 	if err := os.MkdirAll(bootBPath, 0755); err != nil {
 		return err
 	}
 
-	if err := move(hardwareYamlPath, filepath.Join(bootAPath, "hardware.yaml")); err != nil {
-		return err
-	}
-
-	if err := move(kernelPath, filepath.Join(bootAPath, filepath.Base(kernelPath))); err != nil {
-		return err
-	}
-
-	if err := move(initrdPath, filepath.Join(bootAPath, filepath.Base(initrdPath))); err != nil {
-		return err
-	}
-
-	if err := img.provisionDtbs(bootDtbPath); err != nil {
-		return err
+	// populate both A/B
+	for _, path := range []string{bootAPath, bootBPath} {
+		if err := os.MkdirAll(path, 0755); err != nil {
+			return err
+		}
+
+		if err := copyFile(hardwareYamlPath, filepath.Join(path, "hardware.yaml")); err != nil {
+			return err
+		}
+
+		if err := copyFile(kernelPath, filepath.Join(path, filepath.Base(kernelPath))); err != nil {
+			return err
+		}
+
+		if err := copyFile(initrdPath, filepath.Join(path, filepath.Base(initrdPath))); err != nil {
+			return err
+		}
+
+		// create layout
+		bootDtbPath := filepath.Join(path, "dtbs")
+		if err := os.MkdirAll(bootDtbPath, 0755); err != nil {
+			return err
+		}
+
+		if err := img.provisionDtbs(oem, bootDtbPath); err != nil {
+			return err
+		}
 	}
 
 	if err := img.provisionUenv(bootuEnvPath); err != nil {
@@ -374,7 +381,7 @@
 	// if a uEnv.txt is provided in the flashtool-assets, use it
 	if _, err := os.Stat(uEnvPath); err == nil {
 		printOut("Adding uEnv.txt to", bootuEnvPath)
-		if err := move(uEnvPath, bootuEnvPath); err != nil {
+		if err := copyFile(uEnvPath, bootuEnvPath); err != nil {
 			return err
 		}
 	} else {
@@ -384,7 +391,7 @@
 	return nil
 }
 
-func (img CoreUBootImage) provisionDtbs(bootDtbPath string) error {
+func (img CoreUBootImage) provisionDtbs(oem OemDescription, bootDtbPath string) error {
 	dtbsPath := filepath.Join(img.baseMount, img.hardware.Dtbs)
 
 	if _, err := os.Stat(dtbsPath); os.IsNotExist(err) {
@@ -401,16 +408,23 @@
 	dtb := filepath.Join(dtbsPath, fmt.Sprintf("%s.dtb", img.platform))
 
 	// if there is a specific dtb for the platform, copy it.
-	if _, err := os.Stat(dtb); err == nil {
-		dst := filepath.Join(bootDtbPath, filepath.Base(dtb))
-		if err := move(dtb, dst); err != nil {
+	// First look in oem and then in device.
+	if oem.Hardware.Dtb != "" && img.platform != "" {
+		oemDtb := filepath.Join(img.System(), oem.InstallPath(), oem.Hardware.Dtb)
+		dst := filepath.Join(bootDtbPath, filepath.Base(dtb))
+		if err := copyFile(oemDtb, dst); err != nil {
+			return err
+		}
+	} else if _, err := os.Stat(dtb); err == nil {
+		dst := filepath.Join(bootDtbPath, filepath.Base(dtb))
+		if err := copyFile(dtb, dst); err != nil {
 			return err
 		}
 	} else {
 		for _, dtbFi := range dtbFis {
 			src := filepath.Join(dtbsPath, dtbFi.Name())
 			dst := filepath.Join(bootDtbPath, dtbFi.Name())
-			if err := move(src, dst); err != nil {
+			if err := copyFile(src, dst); err != nil {
 				return err
 			}
 		}
@@ -475,7 +489,7 @@
 	return nil
 }
 
-func move(src, dst string) error {
+func copyFile(src, dst string) error {
 	dstFile, err := os.Create(dst)
 	if err != nil {
 		return err
@@ -486,7 +500,6 @@
 	if err != nil {
 		return err
 	}
-	defer os.Remove(src)
 	defer srcFile.Close()
 
 	reader := bufio.NewReader(srcFile)

=== modified file 'sysutils/utils.go'
--- sysutils/utils.go	2015-01-21 23:22:53 +0000
+++ sysutils/utils.go	2015-03-18 03:33:51 +0000
@@ -50,7 +50,9 @@
 	case GiB:
 		size = size * 1024 * 1024 * 1024
 	case GB:
-		size = size * 1000 * 1000 * 1000
+		// The image size will be reduced to fit commercial drives that are
+		// smaller than what they claim, 975 comes from 97.5% of the total size
+		size = size * 1000 * 1000 * 975
 	default:
 		panic("improper sizing unit used")
 	}

=== added file 'sysutils/utils_test.go'
--- sysutils/utils_test.go	1970-01-01 00:00:00 +0000
+++ sysutils/utils_test.go	2015-03-18 03:33:51 +0000
@@ -0,0 +1,50 @@
+package sysutils
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type UtilsTestSuite struct {
+	tmpdir string
+}
+
+var _ = Suite(&UtilsTestSuite{})
+
+func (s *UtilsTestSuite) SetUpTest(c *C) {
+	s.tmpdir = c.MkDir()
+}
+
+func (s *UtilsTestSuite) TestCreateEmptyFileGiB(c *C) {
+	f := filepath.Join(s.tmpdir, "gib")
+	c.Assert(CreateEmptyFile(f, 1, GiB), IsNil)
+
+	fStat, err := os.Stat(f)
+	c.Assert(err, IsNil)
+
+	c.Assert(fStat.Size(), Equals, int64(1024*1024*1024))
+}
+
+func (s *UtilsTestSuite) TestCreateEmptyFileGB(c *C) {
+	f1 := filepath.Join(s.tmpdir, "gb")
+	c.Assert(CreateEmptyFile(f1, 1, GB), IsNil)
+
+	fStat1, err := os.Stat(f1)
+	c.Assert(err, IsNil)
+
+	c.Assert(fStat1.Size(), Equals, int64(1000*1000*975))
+
+	f2 := filepath.Join(s.tmpdir, "gb")
+	c.Assert(CreateEmptyFile(f2, 2, GB), IsNil)
+
+	fStat2, err := os.Stat(f2)
+	c.Assert(err, IsNil)
+
+	c.Assert(fStat2.Size(), Equals, int64(2*1000*1000*975))
+}

=== modified file 'ubuntu-device-flash/core.go'
--- ubuntu-device-flash/core.go	2015-01-19 21:38:25 +0000
+++ ubuntu-device-flash/core.go	2015-03-18 03:33:51 +0000
@@ -319,7 +319,17 @@
 
 	systemPath := img.System()
 
-	if err := img.SetupBoot(); err != nil {
+	if err := coreCmd.install(systemPath); err != nil {
+		return err
+	}
+
+	// check if we installed an oem snap
+	oem, err := loadOem(systemPath)
+	if err != nil {
+		return err
+	}
+
+	if err := img.SetupBoot(oem); err != nil {
 		return err
 	}
 
@@ -327,10 +337,6 @@
 		return err
 	}
 
-	if err := coreCmd.install(systemPath); err != nil {
-		return err
-	}
-
 	if !coreCmd.Cloud {
 		cloudBaseDir := filepath.Join("var", "lib", "cloud")
 
@@ -494,6 +500,31 @@
 	return string(pubKey), err
 }
 
+func loadOem(systemPath string) (oem diskimage.OemDescription, err error) {
+	pkgs, err := filepath.Glob(filepath.Join(systemPath, "/oem/*/*/meta/package.yaml"))
+	if err != nil {
+		return oem, err
+	}
+
+	// checking for len(pkgs) > 2 due to the 'current' symlink
+	if len(pkgs) == 0 {
+		return oem, nil
+	} else if len(pkgs) > 2 || err != nil {
+		return oem, errors.New("too many oem packages installed")
+	}
+
+	f, err := ioutil.ReadFile(pkgs[0])
+	if err != nil {
+		return oem, errors.New("failed to read oem yaml")
+	}
+
+	if err := goyaml.Unmarshal([]byte(f), &oem); err != nil {
+		return oem, errors.New("cannot decode oem yaml")
+	}
+
+	return oem, nil
+}
+
 func extractHWDescription(path string) (hw diskimage.HardwareDescription, err error) {
 	// hack to circumvent https://code.google.com/p/go/issues/detail?id=1435
 	if syscall.Getuid() == 0 {

=== modified file 'ubuntu-device-flash/touch.go'
--- ubuntu-device-flash/touch.go	2015-01-15 13:52:31 +0000
+++ ubuntu-device-flash/touch.go	2015-03-18 03:33:51 +0000
@@ -304,29 +304,17 @@
 	if _, err := adb.Shell("rm -rf /cache/recovery/*.xz /cache/recovery/*.xz.asc"); err != nil {
 		log.Fatal("Cannot cleanup /cache/recovery/ to ensure clean deployment", err)
 	}
-	freeSpace := "unknown"
-	dfCacheCmd := "df -h | grep /android/cache"
-	if free, err := adb.Shell(dfCacheCmd); err != nil {
-		log.Fatal("Unable to retrieve free space on target")
-	} else {
-		//Filesystem Size Used Avail Use% Mounted on
-		free := strings.Fields(free)
-		if len(free) > 3 {
-			freeSpace = free[3]
-		}
-	}
-	errMsg := "Cannot push %s to device: free space on /cache/recovery is %s"
 	for {
 		file := <-files
 		go func() {
 			log.Printf("Start pushing %s to device", file.FilePath)
 			err := adb.Push(file.FilePath, "/cache/recovery/")
 			if err != nil {
-				log.Fatalf(errMsg, file.SigPath, freeSpace)
+				log.Fatal(err)
 			}
 			err = adb.Push(file.SigPath, "/cache/recovery/")
 			if err != nil {
-				log.Fatalf(errMsg, file.SigPath, freeSpace)
+				log.Fatal(err)
 			}
 			log.Printf("Done pushing %s to device", file.FilePath)
 			done <- true