Merge lp:~mvo/ubuntu-cdimage/mvo into lp:ubuntu-cdimage

[Michael Vogt]

[lp:~mvo/ubuntu-cdimage/mvo]

1447. By Michael Vogt on 2014-11-11

merge from lp:ubuntu-cdimage

[Steve Langasek (vorlon)]

Unfortunately this "work in progress" branch is no longer appropriate to merge as-is (e.g. it tries to bzr checkouts of code bases that have moved to git). And now we are moving ubuntu-cdimage to git. So if there is anything here you still want landed, please resubmit it against git.

review: Needs Resubmitting

Unmerged revisions

1447. By Michael Vogt on 2014-11-11

merge from lp:ubuntu-cdimage

1446. By Michael Vogt on 2014-07-24

fix --subproject option

1445. By Michael Vogt on 2014-07-24

HACKING: updated

1444. By Michael Vogt on 2014-07-24

mention ./first-run.sh

1443. By Michael Vogt on 2014-07-24

remove debian-cd mention, thanks to Colin!

1442. By Michael Vogt on 2014-07-24

publish all daily-preinstaled as .tar.gz (not just ubuntu-touch)

1441. By Michael Vogt on 2014-07-23

WIP: UBUNTU_FORCE_LOCAL can create rootfs now with lp:~mvo/launchpad-buildd/mvo

1440. By Michael Vogt on 2014-07-23

merge latest changes

1439. By Michael Vogt on 2014-07-23

HACKING: add info about failures

1438. By Michael Vogt on 2014-07-23

HACKING: use local config instead of example config

=== added file 'HACKING'
--- HACKING	1970-01-01 00:00:00 +0000
+++ HACKING	2014-11-11 12:49:49 +0000
@@ -0,0 +1,22 @@
+Hacking on ubuntu-cdimage
+=========================
+
+
+Livefs based images
+-------------------
+If you want to test build livefs based image you need to run:
+
+$ sudo apt-get install procmail
+  - lockfile is needed from procmail
+$ ./first-run.sh
+
+Run:
+$ export PATH=$(pwd)/bin:$PATH
+create a chroot:
+$ mkdir -p ~/build-ubuntu-core/chroot-autobuild/build
+$ sudo debootstrap utopic ~/build-ubuntu-core/chroot-autobuild/
+$ UBUNTU_FORCE_LOCAL=1 DEBUG=1 for-project ubuntu-core cron.daily-preinstalled --live
+
+
+The resulting image(s) are available under ./scratch
+etc/resolv.conf
\ No newline at end of file

=== added file 'first-run.sh'
--- first-run.sh	1970-01-01 00:00:00 +0000
+++ first-run.sh	2014-11-11 12:49:49 +0000
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+for p in germinate britney ubuntu-archive-tools launchpad-buildd ; do
+    if [ ! -d  $p ]; then
+        bzr branch lp:$p
+    fi
+done
+

=== modified file 'lib/cdimage/build.py'
--- lib/cdimage/build.py	2014-10-10 10:52:42 +0000
+++ lib/cdimage/build.py	2014-11-11 12:49:49 +0000
@@ -186,7 +186,7 @@
     if subprocess.call(
             ["lockfile", "-!", "-l", "43200", "-r", "0", lock]) == 0:
         raise Exception(
-            "%s is unable to start rsync; lock file exists." % fqdn)
+            "%s is unable to start rsync; lock file %s exists." % (fqdn, lock))
     try:
         log_path = os.path.join(config.root, "log", "rsync.log")
         osextras.ensuredir(os.path.dirname(log_path))
@@ -200,7 +200,7 @@
             exclude = env.get("RSYNC_EXCLUDE", "").split()
             source_target = ["%s/" % env["RSYNC_SRC"], "%s/" % target]
 
-            subprocess.call(
+            subprocess.check_call(
                 command_base + [
                     "--exclude", "Packages*", "--exclude", "Sources*",
                     "--exclude", "Release*", "--exclude", "InRelease",
@@ -208,7 +208,7 @@
                 stdout=log, stderr=subprocess.STDOUT, env=env)
 
             # Second pass to update metadata and clean up old files.
-            subprocess.call(
+            subprocess.check_call(
                 command_base + [
                     "--delay-updates", "--delete", "--delete-after",
                 ] + exclude + source_target,
@@ -486,9 +486,10 @@
                     print("tar archive", file=f)
                 shutil.copy2(
                     "%s.manifest" % live_prefix, "%s.manifest" % output_prefix)
-                if config.project == "ubuntu-touch":
+                if config.image_type == "daily-preinstalled":
                     osextras.link_force(
                         "%s.raw" % output_prefix, "%s.tar.gz" % output_prefix)
+                if config.project == "ubuntu-touch":
                     add_android_support(config, arch, output_dir)
                     custom = "%s.custom.tar.gz" % live_prefix
                     if os.path.exists(custom):

=== modified file 'lib/cdimage/livefs.py'
--- lib/cdimage/livefs.py	2014-10-10 10:46:16 +0000
+++ lib/cdimage/livefs.py	2014-11-11 12:49:49 +0000
@@ -158,6 +158,28 @@
 
     return liveproject
 
+def live_build_command_buildlivefs(config, arch):
+    command = ["./launchpad-buildd/buildlivefs"]
+    command.append("--build-id=%s" % live_project(config, arch))
+    if config["UBUNTU_DEFAULTS_LOCALE"]:
+        command.append("--locale=%s" % config["UBUNTU_DEFAULTS_LOCALE"])
+    if config.project in ("ubuntu-core", "ubuntu-touch"):
+        command.append("--image-format=plain")
+    cpuarch, subarch = split_arch(arch)
+    if cpuarch:
+        command.extend(["--arch", cpuarch])
+    if subarch:
+        command.extend(["--subarch", subarch])
+    if config["PROPOSED"]:
+        command.append("--proposed")
+    if config.series:
+        command.extend(["--series", config.series])
+    command.append("--project=%s" % config.project)
+    if config.subproject:
+        command.extend(["--subproject", config.subproject])
+    command.append(live_project(config, arch))
+    return command
+
 
 def live_build_command(config, arch):
     command = [
@@ -354,22 +376,34 @@
             continue
         full_name = live_build_full_name(config, arch)
         timestamp = time.strftime("%F %T")
-        lp, lp_livefs = get_lp_livefs(config, arch)
-        if lp_livefs is None:
-            machine = live_builder(config, arch)
-        else:
-            machine = "Launchpad"
-        logger.info(
-            "%s on %s starting at %s" % (full_name, machine, timestamp))
-        tracker_set_rebuild_status(config, [0, 1], 2, arch)
-        if lp_livefs is not None:
-            lp_kwargs = live_build_lp_kwargs(config, lp, lp_livefs, arch)
-            lp_build = lp_livefs.requestBuild(**lp_kwargs)
-            logger.info("%s: %s" % (full_name, lp_build.web_link))
-            lp_builds.append((lp_build, arch, full_name, machine, None))
-        else:
-            proc = subprocess.Popen(live_build_command(config, arch))
+
+        if os.environ.get("UBUNTU_FORCE_LOCAL", ""):
+            machine = "local"
+            name = live_project(config, arch)
+            basedir = os.path.expanduser(
+                "~/build-%s/chroot-autobuild/build/" % name)
+            config["LIVECD_BASE"] = basedir
+            proc = subprocess.Popen(
+                live_build_command_buildlivefs(config, arch))
             builds[proc.pid] = (proc, arch, full_name, machine)
+        else:
+            lp, lp_livefs = get_lp_livefs(config, arch)
+            if lp_livefs is None:
+                machine = live_builder(config, arch)
+            else:
+                machine = "Launchpad"
+            logger.info(
+                "%s on %s starting at %s" % (full_name, machine, timestamp))
+            tracker_set_rebuild_status(config, [0, 1], 2, arch)
+
+            if lp_livefs is not None:
+                lp_kwargs = live_build_lp_kwargs(config, lp, lp_livefs, arch)
+                lp_build = lp_livefs.requestBuild(**lp_kwargs)
+                logger.info("%s: %s" % (full_name, lp_build.web_link))
+                lp_builds.append((lp_build, arch, full_name, machine, None))
+            else:
+                proc = subprocess.Popen(live_build_command(config, arch))
+                builds[proc.pid] = (proc, arch, full_name, machine)
 
     successful = set()
 
@@ -606,8 +640,11 @@
         liveproject_subarch = "%s-%s" % (liveproject, subarch)
     else:
         liveproject_subarch = liveproject
-
-    lp, lp_livefs = get_lp_livefs(config, arch)
+    
+    if os.environ.get("UBUNTU_FORCE_LOCAL", ""):
+        lp, lp_livefs = None, None
+    else:
+        lp, lp_livefs = get_lp_livefs(config, arch)
     if lp_livefs is not None:
         lp_kwargs = live_build_lp_kwargs(config, lp, lp_livefs, arch)
         lp_build = lp_livefs.getLatestBuild(

=== modified file 'lib/cdimage/osextras.py'
--- lib/cdimage/osextras.py	2014-05-28 09:18:44 +0000
+++ lib/cdimage/osextras.py	2014-11-11 12:49:49 +0000
@@ -144,7 +144,10 @@
         raise FetchError("empty source URL (downloading to %s)" % target)
 
     if source.startswith("/"):
-        os.link(source, target)
+        try:
+            os.link(source, target)
+        except OSError as e:
+            raise FetchError("Can not link %s->%s: %s" % (source, target, e))
         return
 
     # Match lazr.restfulclient, for convenience when working with

=== modified file 'lib/cdimage/tracker.py'
--- lib/cdimage/tracker.py	2014-10-30 10:00:49 +0000
+++ lib/cdimage/tracker.py	2014-11-11 12:49:49 +0000
@@ -17,6 +17,7 @@
 
 """QATracker integration functions."""
 
+import os
 import xmlrpclib
 
 from cdimage.log import logger
@@ -35,6 +36,10 @@
     except ImportError:
         # Become a no-op if the isotracker module can't be found
         return
+    # iso tracker runs sys.exit(1) *inside* the ISOTracker class
+    # if this file does not exist
+    if not os.path.exists(os.path.expanduser('~/.isotracker.conf')):
+        return
 
     if not arches:
         arches = config.arches

=== modified file 'lib/cdimage/tree.py'
--- lib/cdimage/tree.py	2014-10-30 11:30:48 +0000
+++ lib/cdimage/tree.py	2014-11-11 12:49:49 +0000
@@ -1,4 +1,4 @@
-# -*- coding: UTF-8 -*-
+# -*- coding: utf-8 -*-
 
 # Copyright (C) 2012, 2013 Canonical Ltd.
 # Author: Colin Watson <[e-mail]>