Merge lp:~mvo/snappy/snappy-lp1449032-poor-mans-rsync-15.04 into lp:~snappy-dev/snappy/15.04-deprecated

[Michael Vogt]

Only update changed files in SyncBootloader().

This branch changes SyncBootloader() to only update files that actually are different instead of just copying everything everytime. It also fixes some visual progress artefacts. Please let me know if you prefer that in a different branch.

[lp:~mvo/snappy/snappy-lp1449032-poor-mans-rsync-15.04]

445. By Michael Vogt on 2015-05-29

progress/progress.go: fix visual artifacts on update

[John Lenton (chipaca)]

Alas! Needs fixing. Inline.

review: Needs Fixing

[lp:~mvo/snappy/snappy-lp1449032-poor-mans-rsync-15.04]

446. By Michael Vogt on 2015-05-29

address review comments, many thanks to John!

447. By Michael Vogt on 2015-05-29

helpers/helpers.go: add error checking for os.Walk in RSyncWithDelete

448. By Michael Vogt on 2015-05-29

snappy/systemimage.go: only call pb.Notify() if there is a ProgressMeter

449. By Michael Vogt on 2015-05-29

progress/progress.go: do not unset t.pbar in Spin()

450. By Michael Vogt on 2015-05-29

progress/progress.go: make spinner clear its line

[John Lenton (chipaca)]

More problems.

Also tests don't actually pass (you've broken a Spin() test?).

[lp:~mvo/snappy/snappy-lp1449032-poor-mans-rsync-15.04]

451. By Michael Vogt on 2015-05-29

use SkipDir in RSyncWithDelete

452. By Michael Vogt on 2015-05-29

progress/progress_test.go: fix test

453. By Michael Vogt on 2015-05-29

add SyncWithDelete fails test

[Michael Vogt (mvo)]

Thanks very much, really appreciated, see inline reply.

[John Lenton (chipaca)]

review: Approve

[Ricardo Salveti (rsalveti)]

Don't we have auto merge for 15.04? Or is the CI just failing for it?

=== modified file 'helpers/helpers.go'
--- helpers/helpers.go	2015-04-29 09:10:08 +0000
+++ helpers/helpers.go	2015-05-29 13:52:36 +0000
@@ -23,6 +23,7 @@
 	"encoding/hex"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"os/exec"
@@ -317,3 +318,71 @@
 	return syscall.Getuid() == 0 || syscall.Getgid() == 0
 
 }
+
+func makeDirMap(dirName string) (map[string]struct{}, error) {
+	dir, err := ioutil.ReadDir(dirName)
+	if err != nil {
+		return nil, err
+	}
+
+	dirMap := make(map[string]struct{})
+	for _, dent := range dir {
+		if dent.IsDir() {
+			return nil, fmt.Errorf("subdirs are not supported")
+		}
+		dirMap[dent.Name()] = struct{}{}
+	}
+
+	return dirMap, nil
+}
+
+// RSyncWithDelete syncs srcDir to destDir
+func RSyncWithDelete(srcDirName, destDirName string) error {
+
+	// first remove everything thats not in srcdir
+	err := filepath.Walk(destDirName, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// relative to the root "destDirName"
+		relPath := path[len(destDirName):]
+		if !FileExists(filepath.Join(srcDirName, relPath)) {
+			if err := os.RemoveAll(path); err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// then copy or update the data from srcdir to destdir
+	err = filepath.Walk(srcDirName, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// relative to the root "srcDirName"
+		relPath := path[len(srcDirName):]
+		if info.IsDir() {
+			return os.MkdirAll(filepath.Join(destDirName, relPath), info.Mode())
+		}
+		src := path
+		dst := filepath.Join(destDirName, relPath)
+		if !FilesAreEqual(src, dst) {
+			// XXX: on snappy-trunk we can use CopyFile here
+			output, err := exec.Command("cp", "-va", src, dst).CombinedOutput()
+			if err != nil {
+				fmt.Errorf("Failed to copy %s to %s (%s)", src, dst, output)
+			}
+		}
+		return nil
+	})
+
+	return err
+}

=== modified file 'helpers/helpers_test.go'
--- helpers/helpers_test.go	2015-04-23 11:12:32 +0000
+++ helpers/helpers_test.go	2015-05-29 13:52:36 +0000
@@ -19,6 +19,7 @@
 
 import (
 	"compress/gzip"
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -268,3 +269,104 @@
 	c.Assert(err, IsNil)
 	c.Assert(home, Equals, oldHome)
 }
+
+func makeTestFiles(c *C, srcDir, destDir string) {
+	// a new file
+	err := ioutil.WriteFile(filepath.Join(srcDir, "new"), []byte(nil), 0644)
+	c.Assert(err, IsNil)
+
+	// a existing file that needs update
+	err = ioutil.WriteFile(filepath.Join(destDir, "existing-update"), []byte("old-content"), 0644)
+	c.Assert(err, IsNil)
+	err = ioutil.WriteFile(filepath.Join(srcDir, "existing-update"), []byte("some-new-content"), 0644)
+	c.Assert(err, IsNil)
+
+	// existing file that needs no update
+	err = ioutil.WriteFile(filepath.Join(srcDir, "existing-unchanged"), []byte(nil), 0644)
+	c.Assert(err, IsNil)
+	err = exec.Command("cp", "-a", filepath.Join(srcDir, "existing-unchanged"), filepath.Join(destDir, "existing-unchanged")).Run()
+	c.Assert(err, IsNil)
+
+	// a file that needs removal
+	err = ioutil.WriteFile(filepath.Join(destDir, "to-be-deleted"), []byte(nil), 0644)
+	c.Assert(err, IsNil)
+}
+
+func compareDirs(c *C, srcDir, destDir string) {
+	d1, err := exec.Command("ls", "-al", srcDir).CombinedOutput()
+	c.Assert(err, IsNil)
+	d2, err := exec.Command("ls", "-al", destDir).CombinedOutput()
+	c.Assert(err, IsNil)
+	c.Assert(string(d1), Equals, string(d2))
+	// ensure content got updated
+	c1, err := exec.Command("sh", "-c", fmt.Sprintf("find %s -type f |xargs cat", srcDir)).CombinedOutput()
+	c.Assert(err, IsNil)
+	c2, err := exec.Command("sh", "-c", fmt.Sprintf("find %s -type f |xargs cat", destDir)).CombinedOutput()
+	c.Assert(err, IsNil)
+	c.Assert(string(c1), Equals, string(c2))
+}
+
+func (ts *HTestSuite) TestSyncDirs(c *C) {
+
+	for _, l := range [][2]string{
+		[2]string{"src-short", "dst-loooooooooooong"},
+		[2]string{"src-loooooooooooong", "dst-short"},
+		[2]string{"src-eq", "dst-eq"},
+	} {
+
+		// ensure we have src, dest dirs with different length
+		srcDir := filepath.Join(c.MkDir(), l[0])
+		err := os.MkdirAll(srcDir, 0755)
+		c.Assert(err, IsNil)
+		destDir := filepath.Join(c.MkDir(), l[1])
+		err = os.MkdirAll(destDir, 0755)
+		c.Assert(err, IsNil)
+
+		// add a src subdir
+		subdir := filepath.Join(srcDir, "subdir")
+		err = os.Mkdir(subdir, 0755)
+		c.Assert(err, IsNil)
+		makeTestFiles(c, subdir, destDir)
+
+		// add a dst subdir that needs to get deleted
+		subdir2 := filepath.Join(destDir, "to-be-deleted-subdir")
+		err = os.Mkdir(subdir2, 0755)
+		subdir3 := filepath.Join(subdir2, "to-be-deleted-sub-subdir")
+		err = os.Mkdir(subdir3, 0755)
+
+		// and a toplevel
+		makeTestFiles(c, srcDir, destDir)
+
+		// do it
+		err = RSyncWithDelete(srcDir, destDir)
+		c.Assert(err, IsNil)
+
+		// ensure meta-data is identical
+		compareDirs(c, srcDir, destDir)
+		compareDirs(c, filepath.Join(srcDir, "subdir"), filepath.Join(destDir, "subdir"))
+	}
+}
+
+func (ts *HTestSuite) TestSyncDirFails(c *C) {
+	srcDir := c.MkDir()
+	err := os.MkdirAll(srcDir, 0755)
+	c.Assert(err, IsNil)
+
+	destDir := c.MkDir()
+	err = os.MkdirAll(destDir, 0755)
+	c.Assert(err, IsNil)
+
+	err = ioutil.WriteFile(filepath.Join(destDir, "meep"), []byte(nil), 0644)
+	c.Assert(err, IsNil)
+
+	// ensure remove fails
+	err = os.Chmod(destDir, 0100)
+	c.Assert(err, IsNil)
+	// make tempdir cleanup work again
+	defer os.Chmod(destDir, 0755)
+
+	// do it
+	err = RSyncWithDelete(srcDir, destDir)
+	c.Check(err, NotNil)
+	c.Check(err, ErrorMatches, ".*permission denied.*")
+}

=== modified file 'partition/bootloader_uboot.go'
--- partition/bootloader_uboot.go	2015-04-30 11:28:39 +0000
+++ partition/bootloader_uboot.go	2015-05-29 13:52:36 +0000
@@ -218,10 +218,7 @@
 	srcDir := u.currentBootPath
 	destDir := u.otherBootPath
 
-	// always start from scratch: all files here are owned by us.
-	os.RemoveAll(destDir)
-
-	return runCommand("/bin/cp", "-a", srcDir, destDir)
+	return helpers.RSyncWithDelete(srcDir, destDir)
 }
 
 func (u *uboot) HandleAssets() (err error) {

=== modified file 'partition/partition.go'
--- partition/partition.go	2015-03-30 14:56:46 +0000
+++ partition/partition.go	2015-05-29 13:52:36 +0000
@@ -811,6 +811,9 @@
 		return err
 	}
 
+	// XXX: first toggle roofs and then handle assets? that seems
+	//      wrong given that handleAssets may fails and we will
+	//      knowingly boot into a broken system
 	err = p.RunWithOther(RW, func(otherRoot string) (err error) {
 		return bootloader.ToggleRootFS()
 	})

=== modified file 'progress/progress.go'
--- progress/progress.go	2015-04-21 15:52:02 +0000
+++ progress/progress.go	2015-05-29 13:52:36 +0000
@@ -128,8 +128,9 @@
 // Finished stops displaying the progress
 func (t *TextProgress) Finished() {
 	if t.pbar != nil {
-		t.pbar.FinishPrint("Done")
+		t.pbar.Finish()
 	}
+	fmt.Println("Done")
 }
 
 // Write is there so that progress can implment a Writer and can be
@@ -142,7 +143,10 @@
 // that have a unknown duration
 func (t *TextProgress) Spin(msg string) {
 	states := `|/-\`
-	fmt.Printf("\r%s[%c]", msg, states[t.spinStep])
+
+	// clear until end of line
+	clearUntilEOL := "\033[K"
+	fmt.Printf("\r%s[%c]%s", msg, states[t.spinStep], clearUntilEOL)
 	t.spinStep++
 	if t.spinStep >= len(states) {
 		t.spinStep = 0

=== modified file 'progress/progress_test.go'
--- progress/progress_test.go	2015-04-22 10:46:52 +0000
+++ progress/progress_test.go	2015-05-29 13:52:36 +0000
@@ -58,7 +58,7 @@
 	f.Seek(0, 0)
 	progress, err := ioutil.ReadAll(f)
 	c.Assert(err, IsNil)
-	c.Assert(string(progress), Equals, "\rm[|]\rm[/]\rm[-]\rm[\\]\rm[|]\rm[/]")
+	c.Assert(string(progress), Equals, "\rm[|]\x1b[K\rm[/]\x1b[K\rm[-]\x1b[K\rm[\\]\x1b[K\rm[|]\x1b[K\rm[/]\x1b[K")
 }
 
 func (ts *ProgressTestSuite) testAgreed(answer string, value bool, c *C) {

=== modified file 'snappy/systemimage.go'
--- snappy/systemimage.go	2015-04-15 20:39:24 +0000
+++ snappy/systemimage.go	2015-05-29 13:52:36 +0000
@@ -189,6 +189,9 @@
 
 	// Ensure there is always a kernel + initrd to boot with, even
 	// if the update does not provide new versions.
+	if pb != nil {
+		pb.Notify("Syncing boot files")
+	}
 	err = s.partition.SyncBootloaderFiles()
 	if err != nil {
 		return "", err
@@ -217,6 +220,13 @@
 		return "", err
 	}
 
+	// XXX: ToggleNextBoot() calls handleAssets() (but not SyncBootloader
+	//      files :/) - handleAssets() may copy kernel/initramfs to the
+	//      sync mounted /boot/uboot, so its very slow, tell the user
+	//      at least that something is going on
+	if pb != nil {
+		pb.Notify("Updating boot files")
+	}
 	if err = s.partition.ToggleNextBoot(); err != nil {
 		return "", err
 	}

=== modified file 'snappy/systemimage_native.go'
--- snappy/systemimage_native.go	2015-04-02 23:42:34 +0000
+++ snappy/systemimage_native.go	2015-05-29 13:52:36 +0000
@@ -156,6 +156,8 @@
 			pb.Set(genericData.Now)
 		}
 	}
+	// ugly: avoid Spin() artifacts
+	pb.Notify("\nApply done")
 
 	if err := scanner.Err(); err != nil {
 		return err