Merge lp:~jamesodhunt/upstart/async-fix-job-new-destructor into lp:upstart

[James Hunt]

Unmerged revisions

=== modified file 'doc/states.dot'
--- doc/states.dot	2014-03-05 11:17:37 +0000
+++ doc/states.dot	2014-05-22 13:31:23 +0000
@@ -20,44 +20,79 @@
  */
 
 digraph {
-	rankdir=LR;
 	edge [fontsize=10];
 
 	waiting [shape=diamond];
 	starting [label="starting\n(emit starting)"];
+	security_spawning [label="security-spawning"];
 	security [label="security"];
+	pre_starting [label="pre-starting"];
 	pre_start [label="pre-start"];
+	spawning [label="spawning"];
 	spawned [label="spawned\n(wait for pid)"];
+	post_starting [label="post-starting"];
 	post_start [label="post-start"];
 	emit_started [shape=rectangle,label="emit started"];
 	running [shape=diamond];
+	pre_stopping [label="pre-stopping"];
 	pre_stop [label="pre-stop"];
 	stopping [label="stopping\n(emit stopping)"];
 	killed [label="killed\n(wait for SIGCHLD)"];
+	post_stopping [label="post-stopping"];
 	post_stop [label="post-stop"];
 	emit_stopped [shape=rectangle,label="emit stopped"];
 
-	waiting -> starting [color=green];
-	starting -> security [color=green];
-	security -> pre_start [color=green];
+	constraint=false;
+	waiting -> starting [color=green,weight=10];
+
+	starting -> security_spawning [color=green,weight=10];
+	starting -> stopping [color=red];
+
+	security_spawning -> security [color=green,weight=10];
+	security_spawning -> stopping [color=red];
+
+	security -> pre_starting [color=green,weight=10];
 	security -> stopping [color=red];
-	starting -> stopping [color=red];
-	pre_start -> spawned [color=green];
+
+	pre_starting -> pre_start [color=green,weight=10];
+	pre_starting -> stopping [color=red];
+
+	pre_start -> spawning [color=green,weight=10];
 	pre_start -> stopping [color=red];
-	spawned -> post_start [color=green];
+
+	spawning -> spawned [color=green,weight=10];
+	spawning -> stopping [color=red];
+
+	spawned -> post_starting [color=green,weight=10];
 	spawned -> stopping [color=red];
-	post_start -> emit_started -> running [color=green];
+
+	post_starting -> post_start [color=green,weight=10];
+	post_starting -> stopping [color=red];
+
+	post_start -> emit_started -> running [color=green,weight=10];
 	post_start -> stopping [color=red];
-	running -> pre_stop [color=red,label="pid > 0"];
-	running -> stopping [color=red,label="pid == 0"];
-	running -> stopping [color=green,label="respawn"];
+
+	running -> pre_stopping [color=red,fontcolor=red,label="pid > 0",weight=10];
+	running -> stopping [color=red,fontcolor=red,label="pid == 0"];
+	running -> stopping [color=green,fontcolor=darkgreen,label="respawn"];
+
+	pre_stopping -> pre_stop [color=green,weight=10];
+	pre_stopping -> pre_stop [color=red,weight=10];
+
 	pre_stop -> running [color=green];
-	pre_stop -> stopping [color=red];
+	pre_stop -> stopping [color=red,weight=10];
+
 	stopping -> killed [color=green];
 	stopping -> killed [color=red];
-	killed -> post_stop [color=green];
-	killed -> post_stop [color=red];
+
+	killed -> post_stopping [color=green];
+	killed -> post_stopping [color=red];
+
+	post_stopping -> post_stop [color=green,weight=10];
+	post_stopping -> post_stop [color=red];
+
 	post_stop -> starting [color=green];
-	post_stop -> emit_stopped [color=red];
+	post_stop -> emit_stopped [color=red,weight=10];
+
 	emit_stopped -> waiting [color=red];
 }
 
=== removed file 'doc/states.png'
 Binary files doc/states.png	2014-03-05 11:17:37 +0000 and doc/states.png	1970-01-01 00:00:00 +0000 differ
=== modified file 'init/Makefile.am'
--- init/Makefile.am	2013-11-12 14:00:36 +0000
+++ init/Makefile.am	2014-05-22 13:31:23 +0000
@@ -265,6 +265,7 @@
 	session.o log.o state.o xdg.o apparmor.o \
 	com.ubuntu.Upstart.o \
 	com.ubuntu.Upstart.Job.o com.ubuntu.Upstart.Instance.o \
+	$(top_builddir)/test/libtest_util_common.a \
 	$(NIH_LIBS) \
 	$(NIH_DBUS_LIBS) \
 	$(DBUS_LIBS) \
 
=== modified file 'init/job.c'
--- init/job.c	2014-05-07 16:34:44 +0000
+++ init/job.c	2014-05-22 13:31:23 +0000
@@ -70,14 +70,6 @@
 	__attribute__ ((warn_unused_result));
 
 static const char *
-job_state_enum_to_str (JobState state)
-	__attribute__ ((warn_unused_result));
-
-static JobState
-job_state_str_to_enum (const char *state)
-	__attribute__ ((warn_unused_result));
-
-static const char *
 job_trace_state_enum_to_str (TraceState state)
 	__attribute__ ((warn_unused_result));
 
@@ -93,6 +85,31 @@
 job_deserialise_kill_timer (json_object *json)
 	__attribute__ ((warn_unused_result));
 
+int
+job_destroy (Job *job)
+{
+	int  i;
+
+	nih_assert (job);
+
+	/* Free any associated NihIo's to avoid the handlers getting
+	 * called potentially after the job has been freed.
+	 */
+	if (job->process_data) {
+		for (i = 0; i < PROCESS_LAST; i++) {
+			if (job->process_data[i]) {
+				nih_free (job->process_data[i]);
+
+				job->process_data[i] = NULL;
+			}
+		}
+	}
+	nih_list_destroy (&job->entry);
+
+	return 0;
+}
+
+
 /**
  * job_new:
  * @class: class of job,
@@ -125,7 +142,10 @@
 
 	nih_list_init (&job->entry);
 
-	nih_alloc_set_destructor (job, nih_list_destroy);
+	/* Ensure unset before destructor could possibly be called */
+	job->process_data = NULL;
+
+	nih_alloc_set_destructor (job, job_destroy);
 
 	job->name = nih_strdup (job, name);
 	if (! job->name)
@@ -207,6 +227,24 @@
 		job_register (job, conn, TRUE);
 	}
 
+	/* Since some job processes can run in parallel, we must ensure
+	 * that the asynchronous-spawning of such job processes is
+	 * handled by providing a handler for each pid.
+	 *
+	 * Strictly, this is only necessary for certain combinations of
+	 * job processes (such as PROCESS_MAIN and PROCESS_POST_START),
+	 * however for consistency with other entities (such as Log), we
+	 * create a slot for all job processes since there is minimal
+	 * overhead (a single pointer) for those job processes tha
+	 * cannot run in parallel with others.
+	 */
+	job->process_data = nih_alloc (job, sizeof (JobProcessData *) * PROCESS_LAST);
+	if (! job->process_data)
+		goto error;
+
+	for (i = 0; i < PROCESS_LAST; i++)
+		job->process_data[i] = NULL;
+
 	return job;
 
 error:
@@ -398,64 +436,78 @@
 			job->blocker = job_emit_event (job);
 
 			break;
-		case JOB_SECURITY:
+		case JOB_SECURITY_SPAWNING:
 			nih_assert (job->goal == JOB_START);
 			nih_assert (old_state == JOB_STARTING);
 
 			if (job->class->process[PROCESS_SECURITY]
 			    && apparmor_available()) {
-				if (job_process_run (job, PROCESS_SECURITY) < 0) {
-					job_failed (job, PROCESS_SECURITY, -1);
-					job_change_goal (job, JOB_STOP);
-					state = job_next_state (job);
-				}
-			} else {
+			    job_process_start (job, PROCESS_SECURITY);
+			}
+			state = job_next_state (job);
+			break;
+		case JOB_SECURITY:
+			nih_assert (job->goal == JOB_START);
+			nih_assert (old_state == JOB_SECURITY_SPAWNING);
+
+			if (! (job->class->process[PROCESS_SECURITY]
+			       && apparmor_available())) {
 				state = job_next_state (job);
 			}
-
 			break;
-		case JOB_PRE_START:
+		case JOB_PRE_STARTING:
 			nih_assert (job->goal == JOB_START);
 			nih_assert (old_state == JOB_SECURITY);
-
+			/* spawn pre-start asynchronously, child
+			 * watcher asynchronously will change goal to
+			 * stop if spawning fails */
 			if (job->class->process[PROCESS_PRE_START]) {
-				if (job_process_run (job, PROCESS_PRE_START) < 0) {
-					job_failed (job, PROCESS_PRE_START, -1);
-					job_change_goal (job, JOB_STOP);
-					state = job_next_state (job);
-				}
-			} else {
-				state = job_next_state (job);
-			}
+			    job_process_start (job, PROCESS_PRE_START);
+			}	
+			state = job_next_state (job);
+			break;
+		case JOB_PRE_START:
+			nih_assert (job->goal == JOB_START);
+			nih_assert (old_state == JOB_PRE_STARTING);
 
+			/* if no pre-start process, go to next
+			 * state. otherwise async child watcher will
+			 * trigger us to go to the next state */
+			if (! job->class->process[PROCESS_PRE_START])
+			    state = job_next_state (job);
 			break;
-		case JOB_SPAWNED:
+		case JOB_SPAWNING:
 			nih_assert (job->goal == JOB_START);
 			nih_assert (old_state == JOB_PRE_START);
 
 			if (job->class->process[PROCESS_MAIN]) {
-				if (job_process_run (job, PROCESS_MAIN) < 0) {
-					job_failed (job, PROCESS_MAIN, -1);
-					job_change_goal (job, JOB_STOP);
-					state = job_next_state (job);
-				} else if (job->class->expect == EXPECT_NONE)
-					state = job_next_state (job);
-			} else {
-				state = job_next_state (job);
-			}
-
-			break;
-		case JOB_POST_START:
+			    job_process_start (job, PROCESS_MAIN);
+			}
+			state = job_next_state (job);
+			break;
+		case JOB_SPAWNED:
+			nih_assert (job->goal == JOB_START);
+			nih_assert (old_state == JOB_SPAWNING);
+			if (! job->class->process[PROCESS_MAIN]) {
+			    state = job_next_state (job);
+			}
+			break;
+		case JOB_POST_STARTING:
 			nih_assert (job->goal == JOB_START);
 			nih_assert (old_state == JOB_SPAWNED);
 
 			if (job->class->process[PROCESS_POST_START]) {
-				if (job_process_run (job, PROCESS_POST_START) < 0)
-					state = job_next_state (job);
-			} else {
+			    job_process_start (job, PROCESS_POST_START);
+			}
+			state = job_next_state (job);
+			break;
+		case JOB_POST_START:
+			nih_assert (job->goal == JOB_START);
+			nih_assert (old_state == JOB_POST_STARTING);
+
+			if (! job->class->process[PROCESS_POST_START]) {
 				state = job_next_state (job);
 			}
-
 			break;
 		case JOB_RUNNING:
 			nih_assert (job->goal == JOB_START);
@@ -482,20 +534,26 @@
 			}
 
 			break;
-		case JOB_PRE_STOP:
+		case JOB_PRE_STOPPING:
 			nih_assert (job->goal == JOB_STOP);
 			nih_assert (old_state == JOB_RUNNING);
 
 			if (job->class->process[PROCESS_PRE_STOP]) {
-				if (job_process_run (job, PROCESS_PRE_STOP) < 0)
-					state = job_next_state (job);
-			} else {
-				state = job_next_state (job);
+			    job_process_start (job, PROCESS_PRE_STOP);
 			}
+			state = job_next_state (job);
+			break;
+		case JOB_PRE_STOP:
+			nih_assert (job->goal == JOB_STOP);
+			nih_assert (old_state == JOB_PRE_STOPPING);
 
+			if (! job->class->process[PROCESS_PRE_STOP]) {
+			    state = job_next_state (job);
+			}
 			break;
 		case JOB_STOPPING:
 			nih_assert ((old_state == JOB_STARTING)
+				    || (old_state == JOB_PRE_STARTING)
 				    || (old_state == JOB_PRE_START)
 				    || (old_state == JOB_SECURITY)
 				    || (old_state == JOB_SPAWNED)
@@ -508,7 +566,6 @@
 			break;
 		case JOB_KILLED:
 			nih_assert (old_state == JOB_STOPPING);
-
 			if (job->class->process[PROCESS_MAIN]
 			    && (job->pid[PROCESS_MAIN] > 0)) {
 				job_process_kill (job, PROCESS_MAIN);
@@ -517,19 +574,20 @@
 			}
 
 			break;
-		case JOB_POST_STOP:
+		case JOB_POST_STOPPING:
 			nih_assert (old_state == JOB_KILLED);
 
 			if (job->class->process[PROCESS_POST_STOP]) {
-				if (job_process_run (job, PROCESS_POST_STOP) < 0) {
-					job_failed (job, PROCESS_POST_STOP, -1);
-					job_change_goal (job, JOB_STOP);
-					state = job_next_state (job);
-				}
-			} else {
-				state = job_next_state (job);
+			    job_process_start (job, PROCESS_POST_STOP);
 			}
+			state = job_next_state (job);
+			break;
+		case JOB_POST_STOP:
+			nih_assert (old_state == JOB_POST_STOPPING);
 
+			if (! job->class->process[PROCESS_POST_STOP]) {
+			    state = job_next_state (job);
+			}
 			break;
 		case JOB_WAITING:
 			nih_assert (job->goal == JOB_STOP);
@@ -617,6 +675,15 @@
 		case JOB_STOP:
 			return JOB_STOPPING;
 		case JOB_START:
+			return JOB_SECURITY_SPAWNING;
+		default:
+			nih_assert_not_reached ();
+		}
+	case JOB_SECURITY_SPAWNING:
+		switch (job->goal) {
+		case JOB_STOP:
+			return JOB_STOPPING;
+		case JOB_START:
 			return JOB_SECURITY;
 		default:
 			nih_assert_not_reached ();
@@ -626,6 +693,15 @@
 		case JOB_STOP:
 			return JOB_STOPPING;
 		case JOB_START:
+			return JOB_PRE_STARTING;
+		default:
+			nih_assert_not_reached ();
+		}
+	case JOB_PRE_STARTING:
+		switch (job->goal) {
+		case JOB_STOP:
+			return JOB_STOPPING;
+		case JOB_START:
 			return JOB_PRE_START;
 		default:
 			nih_assert_not_reached ();
@@ -635,6 +711,15 @@
 		case JOB_STOP:
 			return JOB_STOPPING;
 		case JOB_START:
+			return JOB_SPAWNING;
+		default:
+			nih_assert_not_reached ();
+		}
+	case JOB_SPAWNING:
+		switch (job->goal) {
+		case JOB_STOP:
+			return JOB_STOPPING;
+		case JOB_START:
 			return JOB_SPAWNED;
 		default:
 			nih_assert_not_reached ();
@@ -644,6 +729,15 @@
 		case JOB_STOP:
 			return JOB_STOPPING;
 		case JOB_START:
+			return JOB_POST_STARTING;
+		default:
+			nih_assert_not_reached ();
+		}
+	case JOB_POST_STARTING:
+		switch (job->goal) {
+		case JOB_STOP:
+			return JOB_STOPPING;
+		case JOB_START:
 			return JOB_POST_START;
 		default:
 			nih_assert_not_reached ();
@@ -665,7 +759,7 @@
 		case JOB_STOP:
 			if (job->class->process[PROCESS_MAIN]
 			    && (job->pid[PROCESS_MAIN] > 0)) {
-				return JOB_PRE_STOP;
+				return JOB_PRE_STOPPING;
 			} else {
 				return JOB_STOPPING;
 			}
@@ -674,6 +768,15 @@
 		default:
 			nih_assert_not_reached ();
 		}
+	case JOB_PRE_STOPPING:
+		switch (job->goal) {
+		case JOB_STOP:
+			return JOB_PRE_STOP;
+		case JOB_START:
+			return JOB_PRE_STOP;
+		default:
+			nih_assert_not_reached ();
+		}
 	case JOB_PRE_STOP:
 		switch (job->goal) {
 		case JOB_STOP:
@@ -698,6 +801,15 @@
 	case JOB_KILLED:
 		switch (job->goal) {
 		case JOB_STOP:
+			return JOB_POST_STOPPING;
+		case JOB_START:
+			return JOB_POST_STOPPING;
+		default:
+			nih_assert_not_reached ();
+		}
+	case JOB_POST_STOPPING:
+		switch (job->goal) {
+		case JOB_STOP:
 			return JOB_POST_STOP;
 		case JOB_START:
 			return JOB_POST_STOP;
@@ -870,8 +982,8 @@
 
 
 /**
- * job_emit_event:
- * @job: job generating the event.
+ * job_emit_event_with_state:
+ * @job: job generating the event,
  *
  * Called from a state change because it believes an event should be
  * emitted.  Constructs the event with the right arguments and environment
@@ -1100,22 +1212,34 @@
 		return N_("waiting");
 	case JOB_STARTING:
 		return N_("starting");
+	case JOB_SECURITY_SPAWNING:
+		return N_("security-spawning");
 	case JOB_SECURITY:
 		return N_("security");
+	case JOB_PRE_STARTING:
+		return N_("pre-starting");
 	case JOB_PRE_START:
 		return N_("pre-start");
+	case JOB_SPAWNING:
+		return N_("spawning");
 	case JOB_SPAWNED:
 		return N_("spawned");
+	case JOB_POST_STARTING:
+		return N_("post-starting");
 	case JOB_POST_START:
 		return N_("post-start");
 	case JOB_RUNNING:
 		return N_("running");
+	case JOB_PRE_STOPPING:
+		return N_("pre-stopping");
 	case JOB_PRE_STOP:
 		return N_("pre-stop");
 	case JOB_STOPPING:
 		return N_("stopping");
 	case JOB_KILLED:
 		return N_("killed");
+	case JOB_POST_STOPPING:
+		return N_("post-stopping");
 	case JOB_POST_STOP:
 		return N_("post-stop");
 	default:
@@ -1140,22 +1264,34 @@
 		return JOB_WAITING;
 	} else if (! strcmp (state, "starting")) {
 		return JOB_STARTING;
+	} else if (! strcmp (state, "security-spawning")) {
+		return JOB_SECURITY_SPAWNING;
 	} else if (! strcmp (state, "security")) {
 		return JOB_SECURITY;
+	} else if (! strcmp (state, "pre-starting")) {
+		return JOB_PRE_STARTING;
 	} else if (! strcmp (state, "pre-start")) {
 		return JOB_PRE_START;
+	} else if (! strcmp (state, "spawning")) {
+		return JOB_SPAWNING;
 	} else if (! strcmp (state, "spawned")) {
 		return JOB_SPAWNED;
+	} else if (! strcmp (state, "post-starting")) {
+		return JOB_POST_STARTING;
 	} else if (! strcmp (state, "post-start")) {
 		return JOB_POST_START;
 	} else if (! strcmp (state, "running")) {
 		return JOB_RUNNING;
+	} else if (! strcmp (state, "pre-stopping")) {
+		return JOB_PRE_STOPPING;
 	} else if (! strcmp (state, "pre-stop")) {
 		return JOB_PRE_STOP;
 	} else if (! strcmp (state, "stopping")) {
 		return JOB_STOPPING;
 	} else if (! strcmp (state, "killed")) {
 		return JOB_KILLED;
+	} else if (! strcmp (state, "post-stopping")) {
+		return JOB_POST_STOPPING;
 	} else if (! strcmp (state, "post-stop")) {
 		return JOB_POST_STOP;
 	} else {
@@ -1626,6 +1762,7 @@
 	json_object      *json_pid;
 	json_object      *json_fds;
 	json_object      *json_logs;
+	json_object      *json_handler_data;
 
 	nih_assert (job);
 
@@ -1771,6 +1908,33 @@
 
 	json_object_object_add (json, "log", json_logs);
 
+	json_handler_data = json_object_new_array ();
+
+	if (! json_handler_data)
+		return json;
+
+	for (int process = 0; process < PROCESS_LAST; process++) {
+		json_object *json_data = NULL;
+
+		/* Only bother serialising if the process data hasn't
+		 * been handled yet.
+		 */
+		if (job->process_data[process] && job->process_data[process]->valid) {
+
+			json_data = job_process_data_serialise (job, 
+					job->process_data[process]);
+
+			if (! json_data)
+				goto error;
+
+		}
+
+		if (json_object_array_add (json_handler_data, json_data) < 0)
+			goto error;
+	}
+
+	json_object_object_add (json, "process_data", json_handler_data);
+
 	return json;
 
 error:
@@ -1834,6 +1998,7 @@
 	json_object    *json_fds;
 	json_object    *json_pid;
 	json_object    *json_logs;
+	json_object    *json_process_data;
 	json_object    *json_stop_on = NULL;
 	size_t          len;
 	int             ret;
@@ -2062,6 +2227,38 @@
 		}
 	}
 
+
+	if (json_object_object_get_ex (json, "process_data", &json_process_data)) {
+		if (! state_check_json_type (json_process_data, array))
+			goto error;
+
+		for (int process = 0; process < PROCESS_LAST; process++) {
+			json_object  *json_data = NULL;
+
+			json_data = json_object_array_get_idx (json_process_data, process);
+
+			if (json_data) {
+				/* NULL if there was no process_data for this job process, or we failed to
+				 * deserialise it; either way, this should be non-fatal.
+				 */
+				job->process_data[process] =
+					job_process_data_deserialise (job->process_data, job, json_data);
+
+				if (! job->process_data[process])
+					goto error;
+
+				/* Recreate watch */
+				if (job->process_data[process]->valid) {
+					job_register_child_handler (job->process_data[process],
+							job->process_data[process]->job_process_fd,
+							job->process_data[process]);
+				}
+			} else {
+				job->process_data[process] = NULL;
+			}
+		}
+	}
+
 	return job;
 
 error:
@@ -2163,19 +2360,25 @@
  *
  * Returns: string representation of @state, or NULL if not known.
  **/
-static const char *
+const char *
 job_state_enum_to_str (JobState state)
 {
 	state_enum_to_str (JOB_WAITING, state);
 	state_enum_to_str (JOB_STARTING, state);
+	state_enum_to_str (JOB_SECURITY_SPAWNING, state);
 	state_enum_to_str (JOB_SECURITY, state);
+	state_enum_to_str (JOB_PRE_STARTING, state);
 	state_enum_to_str (JOB_PRE_START, state);
+	state_enum_to_str (JOB_SPAWNING, state);
 	state_enum_to_str (JOB_SPAWNED, state);
+	state_enum_to_str (JOB_POST_STARTING, state);
 	state_enum_to_str (JOB_POST_START, state);
 	state_enum_to_str (JOB_RUNNING, state);
+	state_enum_to_str (JOB_PRE_STOPPING, state);
 	state_enum_to_str (JOB_PRE_STOP, state);
 	state_enum_to_str (JOB_STOPPING, state);
 	state_enum_to_str (JOB_KILLED, state);
+	state_enum_to_str (JOB_POST_STOPPING, state);
 	state_enum_to_str (JOB_POST_STOP, state);
 
 	return NULL;
@@ -2190,19 +2393,25 @@
  *
  * Returns: JobState representation of @state, or -1 if not known.
  **/
-static JobState
+JobState
 job_state_str_to_enum (const char *state)
 {
 	state_str_to_enum (JOB_WAITING, state);
 	state_str_to_enum (JOB_STARTING, state);
+	state_str_to_enum (JOB_SECURITY_SPAWNING, state);
 	state_str_to_enum (JOB_SECURITY, state);
+	state_str_to_enum (JOB_PRE_STARTING, state);
 	state_str_to_enum (JOB_PRE_START, state);
+	state_str_to_enum (JOB_SPAWNING, state);
 	state_str_to_enum (JOB_SPAWNED, state);
+	state_str_to_enum (JOB_POST_STARTING, state);
 	state_str_to_enum (JOB_POST_START, state);
 	state_str_to_enum (JOB_RUNNING, state);
+	state_str_to_enum (JOB_PRE_STOPPING, state);
 	state_str_to_enum (JOB_PRE_STOP, state);
 	state_str_to_enum (JOB_STOPPING, state);
 	state_str_to_enum (JOB_KILLED, state);
+	state_str_to_enum (JOB_POST_STOPPING, state);
 	state_str_to_enum (JOB_POST_STOP, state);
 
 	return -1;
@@ -2358,3 +2567,58 @@
 error:
 	return NULL;
 }
+
+/**
+ * job_child_error_handler:
+ *
+ * @job: job,
+ * @process: process that failed to start.
+ *
+ * JobProcessErrorHandler that deals with errors resulting from
+ * a failure to start a job process.
+ **/
+void
+job_child_error_handler (Job *job, ProcessType process)
+{
+	nih_assert (job);
+	nih_assert (process > PROCESS_INVALID);
+	nih_assert (process < PROCESS_LAST);
+
+	job->pid[process] = 0;
+
+	switch (process) {
+	case PROCESS_SECURITY:
+		job_failed (job, PROCESS_SECURITY, -1);
+		job_change_goal (job, JOB_STOP);
+		break;
+
+	case PROCESS_PRE_START:
+		job_failed (job, PROCESS_PRE_START, -1);
+		job_change_goal (job, JOB_STOP);
+		job_change_state (job, job_next_state (job));
+		break;
+
+	case PROCESS_MAIN:
+		job_failed (job, PROCESS_MAIN, -1);
+		job_change_goal (job, JOB_STOP);
+		job_change_state (job, job_next_state (job));
+		break;
+
+	case PROCESS_POST_START:
+		job_change_state (job, job_next_state (job));
+		break;
+
+	case PROCESS_PRE_STOP:
+		job_change_state (job, job_next_state (job));
+		break;
+
+	case PROCESS_POST_STOP:
+		job_failed (job, PROCESS_POST_STOP, -1);
+		job_change_goal (job, JOB_STOP);
+		job_change_state (job, job_next_state (job));
+		break;
+
+	default:
+		nih_assert_not_reached ();
+	}
+}
 
=== modified file 'init/job.h'
--- init/job.h	2013-07-22 01:24:26 +0000
+++ init/job.h	2014-05-22 13:31:23 +0000
@@ -1,6 +1,6 @@
 /* upstart
  *
- * Copyright © 2010 Canonical Ltd.
+ * Copyright  2010 Canonical Ltd.
  * Author: Scott James Remnant <[e-mail]>.
  *
  * This program is free software; you can redistribute it and/or modify
@@ -66,14 +66,20 @@
 typedef enum job_state {
 	JOB_WAITING,
 	JOB_STARTING,
+	JOB_SECURITY_SPAWNING,
 	JOB_SECURITY,
+	JOB_PRE_STARTING,
 	JOB_PRE_START,
+	JOB_SPAWNING,
 	JOB_SPAWNED,
+	JOB_POST_STARTING,
 	JOB_POST_START,
 	JOB_RUNNING,
+	JOB_PRE_STOPPING,
 	JOB_PRE_STOP,
 	JOB_STOPPING,
 	JOB_KILLED,
+	JOB_POST_STOPPING,
 	JOB_POST_STOP
 } JobState;
 
@@ -91,6 +97,7 @@
 	TRACE_NORMAL
 } TraceState;
 
+typedef struct job_process_data JobProcessData;
 
 /**
  * Job:
@@ -119,49 +126,102 @@
  * @respawn_count: number of respawns since @respawn_time,
  * @trace_forks: number of forks traced,
  * @trace_state: state of trace,
- * @log: pointer to array of log objects for handling job output.
+ * @log: pointer to array of log objects for handling job output,
+ * @process_data: transitory async job process metadata.
  *
  * This structure holds the state of an active job instance being tracked
  * by the init daemon, the configuration details of the job are available
  * from the @class member.
  **/
 typedef struct job {
-	NihList         entry;
-
-	char           *name;
-	JobClass       *class;
-	char           *path;
-
-	JobGoal         goal;
-	JobState        state;
-	char          **env;
-
-	char          **start_env;
-	char          **stop_env;
-	EventOperator  *stop_on;
-
-	int            *fds;
-	size_t          num_fds;
-
-	pid_t          *pid;
-	Event          *blocker;
-	NihList         blocking;
-
-	NihTimer       *kill_timer;
-	ProcessType     kill_process;
-
-	int             failed;
-	ProcessType     failed_process;
-	int             exit_status;
-
-	time_t          respawn_time;
-	int             respawn_count;
-
-	int             trace_forks;
-	TraceState      trace_state;
-	Log           **log;
+	NihList          entry;
+
+	char            *name;
+	JobClass        *class;
+	char            *path;
+
+	JobGoal          goal;
+	JobState         state;
+	char           **env;
+
+	char           **start_env;
+	char           **stop_env;
+	EventOperator   *stop_on;
+
+	int             *fds;
+	size_t           num_fds;
+
+	pid_t           *pid;
+	Event           *blocker;
+	NihList          blocking;
+
+	NihTimer        *kill_timer;
+	ProcessType      kill_process;
+
+	int              failed;
+	ProcessType      failed_process;
+	int              exit_status;
+
+	time_t           respawn_time;
+	int              respawn_count;
+
+	int              trace_forks;
+	TraceState       trace_state;
+	Log            **log;
+	JobProcessData **process_data;
+
 } Job;
 
+/**
+ * JobProcessData:
+ *
+ * @job: job,
+ * @process: job process to run,
+ * @script: optional script that job should run with,
+ * @shell_fd: file descriptor attached to child that @script should be
+ * written to (or -1 when @script is NULL),
+ * @job_process_fd: open readable file descriptor attached to the child
+ *  process used to detect child setup errors,
+ * @status: exit status or signal in higher byte (iff 
+ *  job_process_terminated() ran before the job_register_child_handler()
+ *  handlers),
+ * @valid: FALSE once the data has been handled, else TRUE.
+ *
+ * Used to keep track of asynchronously started job processes.
+ **/
+typedef struct job_process_data {
+	Job           *job;
+	ProcessType    process;
+	char          *script;
+	int            shell_fd;
+	int            job_process_fd;
+	int            status;
+	int            valid;
+} JobProcessData;
+
+/**
+ * job_register_child_handler:
+ *
+ * @_parent: parent pointer,
+ * @_fd: file descriptor to monitor,
+ * @_data: data to pass to nih_io_reopen().
+ *
+ * Register a watcher to monitor the job process child error file
+ * descriptor @_fd.
+ **/
+#define job_register_child_handler(_parent, _fd, _data) \
+	 while (! nih_io_reopen (_parent, _fd, NIH_IO_STREAM, \
+			(NihIoReader)job_process_child_reader, \
+			(NihIoCloseHandler)job_process_close_handler, \
+			NULL, \
+			_data)) { \
+		NihError *err; \
+		err = nih_error_get (); \
+		if (err->number != ENOMEM) \
+			nih_assert_not_reached (); \
+		nih_free (err); \
+	 }
+
 
 NIH_BEGIN_EXTERN
 
@@ -179,7 +239,6 @@
 
 Event      *job_emit_event      (Job *job);
 
-
 const char *job_name            (Job *job);
 
 const char *job_goal_name       (JobGoal goal)
@@ -228,6 +287,16 @@
 				 const char     *job_name)
 	__attribute__ ((warn_unused_result));
 
+const char *
+job_state_enum_to_str (JobState state)
+	__attribute__ ((warn_unused_result));
+
+JobState
+job_state_str_to_enum (const char *state)
+	__attribute__ ((warn_unused_result));
+
+void job_child_error_handler (Job *job, ProcessType process);
+
 NIH_END_EXTERN
 
 #endif /* INIT_JOB_H */
 
=== modified file 'init/job_class.c'
--- init/job_class.c	2014-05-07 16:34:44 +0000
+++ init/job_class.c	2014-05-22 13:31:23 +0000
@@ -2584,14 +2584,14 @@
 				if (fd < 0)
 					continue;
 
-				if (state_toggle_cloexec (fd, FALSE) < 0)
+				if (state_modify_cloexec (fd, FALSE) < 0)
 					goto error;
 
 				fd = log->fd;
 				if (fd < 0)
 					continue;
 
-				if (state_toggle_cloexec (fd, FALSE) < 0)
+				if (state_modify_cloexec (fd, FALSE) < 0)
 					goto error;
 			}
 		}
 
=== modified file 'init/job_process.c'
--- init/job_process.c	2014-01-31 23:45:02 +0000
+++ init/job_process.c	2014-05-22 13:31:23 +0000
@@ -111,8 +111,6 @@
 static void job_process_error_abort     (int fd, JobProcessErrorType type,
 					 int arg)
 	__attribute__ ((noreturn));
-static int  job_process_error_read      (int fd)
-	__attribute__ ((warn_unused_result));
 static void job_process_remap_fd        (int *fd, int reserved_fd, int error_fd);
 
 /**
@@ -134,7 +132,7 @@
 /* Prototypes for static functions */
 static void job_process_kill_timer      (Job *job, NihTimer *timer);
 static void job_process_terminated      (Job *job, ProcessType process,
-					 int status);
+					 int status, int state_only);
 static int  job_process_catch_runaway   (Job *job);
 static void job_process_stopped         (Job *job, ProcessType process);
 static void job_process_trace_new       (Job *job, ProcessType process);
@@ -149,8 +147,10 @@
 extern int           session_end;
 extern time_t        quiesce_phase_time;
 
+
 /**
- * job_process_run:
+ * job_process_start:
+ *
  * @job: job context for process to be run in,
  * @process: job process to run.
  *
@@ -174,27 +174,33 @@
  * In either case the shell is run with the -e option so that commands will
  * fail if their exit status is not checked.
  *
- * This function will block until the job_process_spawn() call succeeds or
- * a non-temporary error occurs (such as file not found).  It is up to the
- * called to decide whether non-temporary errors are a reason to change the
- * job state or not.
+ * This function will only block on temporary job_process_spawn_with_fd() error
+ * (for example fork() failure): in normal operation it returns as soon as
+ * the fork() was successful, registering an NihIo which provides
+ * asynchronous handling of the child setup stage. The specified error
+ * handler will be called should child setup fail.
  *
- * Returns: zero on success, negative value on non-temporary error.
+ * It is up to the caller to decide whether non-temporary errors are a reason
+ * to change the job state or not.
  **/
-int
-job_process_run (Job         *job,
-		 ProcessType  process)
+void
+job_process_start (Job                    *job,
+		   ProcessType             process)
 {
-	Process         *proc;
-	nih_local char **argv = NULL;
-	nih_local char **env = NULL;
-	nih_local char  *script = NULL;
-	char           **e;
-	size_t           argc, envc;
-	int              fds[2] = { -1, -1 };
-	int              error = FALSE, trace = FALSE, shell = FALSE;
+	Process            *proc;
+	nih_local char    **argv = NULL;
+	nih_local char    **env = NULL;
+	nih_local char     *script = NULL;
+	char              **e;
+	size_t              argc, envc;
+	int                 fds[2] = { -1, -1 };
+	int                 trace = FALSE, shell = FALSE;
+	int                 job_process_fd = -1;
+	JobProcessData     *process_data = NULL;
 
-	nih_assert (job != NULL);
+	nih_assert (job);
+	nih_assert (process > PROCESS_INVALID);
+	nih_assert (process < PROCESS_LAST);
 
 	proc = job->class->process[process];
 	nih_assert (proc != NULL);
@@ -304,35 +310,14 @@
 		trace = TRUE;
 
 	/* Spawn the process, repeat until fork() works */
-	while ((job->pid[process] = job_process_spawn (job, argv, env,
-					trace, fds[0], process)) < 0) {
+	while ((job->pid[process] = job_process_spawn_with_fd (job, argv, env,
+					trace, fds[0], process, &job_process_fd)) < 0) {
 		NihError *err;
 
 		err = nih_error_get ();
-		if (err->number == JOB_PROCESS_ERROR) {
-			/* Non-temporary error condition, we're not going
-			 * to be able to spawn this process.  Clean up after
-			 * ourselves before returning.
-			 */
-			if (shell) {
-				close (fds[0]);
-				close (fds[1]);
-			}
-
-			job->pid[process] = 0;
-
-			/* Return non-temporary error condition */
-			nih_warn (_("Failed to spawn %s %s process: %s"),
-				  job_name (job), process_name (process),
-				  err->message);
-			nih_free (err);
-			return -1;
-		} else if (! error)
-			nih_warn ("%s: %s", _("Temporary process spawn error"),
-				  err->message);
+		nih_warn ("%s: %s", _("Temporary process spawn error"),
+				err->message);
 		nih_free (err);
-
-		error = TRUE;
 	}
 
 	nih_info (_("%s %s process (%d)"),
@@ -341,51 +326,50 @@
 	job->trace_forks = 0;
 	job->trace_state = trace ? TRACE_NEW : TRACE_NONE;
 
-	/* Feed the script to the child process */
 	if (shell) {
-		NihIo *io;
-
 		/* Clean up and close the reading end (we don't need it) */
 		close (fds[0]);
-
-		/* Put the entire script into an NihIo send buffer and
-		 * then mark it for closure so that the shell gets EOF
-		 * and the structure gets cleaned up automatically.
-		 */
-		while (! (io = nih_io_reopen (job, fds[1], NIH_IO_STREAM,
-					      NULL, NULL, NULL, NULL))) {
-			NihError *err;
-
-			err = nih_error_get ();
-			if (err->number != ENOMEM)
-				nih_assert_not_reached ();
-			nih_free (err);
-		}
-
-		/* We're feeding using a pipe, which has a file descriptor
-		 * on the child end even though it open()s it again using
-		 * a path. Instruct the shell to close this extra fd and
-		 * not to leak it.
-		 */
-		NIH_ZERO (nih_io_printf (io, "exec %d<&-\n",
-					 JOB_PROCESS_SCRIPT_FD));
-
-		NIH_ZERO (nih_io_write (io, script, strlen (script)));
-		nih_io_shutdown (io);
-	}
-
-	return 0;
+	}
+
+	/* At this stage, a child process has been spawned, but may not
+	 * yet have been setup appropriately. This operation is handled
+	 * asynchronously since some setup operations may block,
+	 * and even though they are running in a new child process, they could
+	 * still block PID 1 since the latter needs to wait until the
+	 * child is fully setup and the appropriate program has been
+	 * exec'd before marking the job as running.
+	 *
+	 * This is now achieved by creating an NihIo and registering
+	 * appropriate handlers such that notification of successful
+	 * child setup and child error scenarios are handled
+	 * automatically.
+	 *
+	 * job_process_child_reader() handles updating the jobs state.
+	 */
+	process_data = job->process_data[process] =
+		NIH_MUST (job_process_data_new
+				(job->process_data, job, process, job_process_fd));
+
+	if (shell) {
+		process_data->shell_fd = fds[1];
+		process_data->script = script;
+		nih_ref (script, process_data);
+	}
+
+	/* Set up a handler to monitor the child fd asynchronously */
+	job_register_child_handler (job->process_data[process],
+			job_process_fd, process_data);
 }
 
-
 /**
- * job_process_spawn:
+ * job_process_spawn_with_fd:
  * @job: job of process to be spawned,
  * @argv: NULL-terminated list of arguments for the process,
  * @env: NULL-terminated list of environment variables for the process,
  * @trace: whether to trace this process,
  * @script_fd: script file descriptor,
- * @process: job process to spawn.
+ * @process: job process to spawn,
+ * @job_process_fd: readable child setup pipe file descriptor.
  *
  * This function spawns a new process using the class details in @job to set up
  * the environment for it; the process is always a session and process group
@@ -406,26 +390,30 @@
  *
  * This function only spawns the process, it is up to the caller to ensure
  * that the information is saved into the job and that the process is watched,
- * etc.
+ * etc. Also, it is up to the caller to monitor @job_process_fd to
+ * ensure the child setup was successful; data available to read
+ * indicates a failure to setup the child environment (represented by a
+ * JOB_PROCESS_ERROR error) whereas if the descriptor is simply
+ * closed setup was successful and the caller can then mark the job
+ * process as started.
  *
  * Spawning a process may fail for temporary reasons, usually due to a failure
- * of the fork() syscall or communication with the child; or more permanent
- * reasons such as a failure to setup the child environment.  These latter
- * are always represented by a JOB_PROCESS_ERROR error.
+ * of the fork() syscall.
  *
  * Returns: process id of new process on success, -1 on raised error
  **/
 pid_t
-job_process_spawn (Job          *job,
+job_process_spawn_with_fd (Job          *job,
 		   char * const  argv[],
 		   char * const *env,
 		   int           trace,
 		   int           script_fd,
-		   ProcessType   process)
+		   ProcessType   process,
+		   int          *job_process_fd)
 {
 	sigset_t        child_set, orig_set;
 	pid_t           pid;
-	int             i, fds[2];
+	int             i, fds[2] = { -1, -1 };
 	int             pty_master = -1;
 	int             pty_slave = -1;
 	char            pts_name[PATH_MAX];
@@ -438,7 +426,6 @@
 	struct passwd   *pwd = NULL;
 	struct group    *grp = NULL;
 
-
 	nih_assert (job != NULL);
 	nih_assert (job->class != NULL);
 	nih_assert (job->log != NULL);
@@ -478,7 +465,7 @@
 			nih_free (err);
 			close (fds[0]);
 			close (fds[1]);
-			nih_return_no_memory_error(-1);
+			nih_return_no_memory_error (-1);
 		}
 
 		pty_master = posix_openpt (O_RDWR | O_NOCTTY);
@@ -538,23 +525,10 @@
 		sigprocmask (SIG_SETMASK, &orig_set, NULL);
 		close (fds[1]);
 
-		/* Read error from the pipe, return if one is raised */
-		if (job_process_error_read (fds[0]) < 0) {
-			if (class->console == CONSOLE_LOG) {
-				/* Ensure the pty_master watch gets
-				 * removed and the fd closed.
-				 */
-				nih_free (job->log[process]);
-				job->log[process] = NULL;
-			}
-			close (fds[0]);
-			return -1;
-		}
-
-		/* Note that pts_master is closed automatically in the parent when the
-		 * log object is destroyed.
-		 */
-		close (fds[0]);
+		*job_process_fd = fds[0];
+
+		nih_io_set_cloexec (*job_process_fd);
+
 		return pid;
 	} else if (pid < 0) {
 		nih_error_raise_system ();
@@ -923,6 +897,10 @@
 	 *   call exec below.
 	 */
 	if (class->debug) {
+		/* Since we have not exec'd at this point, we will still
+		 * have a copy of the parents fds open. As such, re-exec
+		 * will not work.
+		 */
 		close (fds[1]);
 		raise (SIGSTOP);
 	}
@@ -985,9 +963,11 @@
 	exit (255);
 }
 
+
 /**
- * job_process_error_read:
- * @fd: reading end of pipe.
+ * job_process_error_handler:
+ * @buf: data read from child process,
+ * @len: length of data read as @wire_err.
  *
  * Read from the reading end of the pipe specified by @fd, if we receive
  * data then the child raised a process error which we reconstruct and raise
@@ -996,30 +976,26 @@
  * The reconstructed error will be of JOB_PROCESS_ERROR type, the human-
  * readable message is generated according to the type of process error
  * and argument passed along with it.
- *
- * Returns: zero if no error was found, or negative value on raised error.
  **/
-static int
-job_process_error_read (int fd)
+void
+job_process_error_handler (const char *buf, size_t len)
 {
-	JobProcessWireError  wire_err;
-	ssize_t              len;
-	JobProcessError     *err;
-	const char          *res;
+	JobProcessWireError  *wire_err;
+	JobProcessError      *err;
+	const char           *res;
+
+	wire_err = (JobProcessWireError *)buf;
+
+	nih_assert (wire_err);
 
 	/* Read the error from the pipe; a zero read indicates that the
 	 * exec succeeded so we return success, otherwise if we don't receive
 	 * a JobProcessWireError structure, we return a temporary error so we
 	 * try again.
 	 */
-	len = read (fd, &wire_err, sizeof (wire_err));
-	if (len == 0) {
-		return 0;
-	} else if (len < 0) {
-		nih_return_system_error (-1);
-	} else if (len != sizeof (wire_err)) {
+	if (len != sizeof (JobProcessWireError)) {
 		errno = EILSEQ;
-		nih_return_system_error (-1);
+		nih_return_system_error ();
 	}
 
 	/* Construct a JobProcessError to be raised containing information
@@ -1028,9 +1004,9 @@
 	 */
 	err = NIH_MUST (nih_new (NULL, JobProcessError));
 
-	err->type = wire_err.type;
-	err->arg = wire_err.arg;
-	err->errnum = wire_err.errnum;
+	err->type = wire_err->type;
+	err->arg = wire_err->arg;
+	err->errnum = wire_err->errnum;
 
 	err->error.number = JOB_PROCESS_ERROR;
 
@@ -1218,7 +1194,6 @@
 	}
 
 	nih_error_raise_error (&err->error);
-	return -1;
 }
 
 
@@ -1332,6 +1307,7 @@
 {
 	nih_assert (job);
 	nih_assert (timeout);
+	nih_assert (job->kill_timer == NULL);
 
 	job->kill_process = process;
 	job->kill_timer = NIH_MUST (nih_timer_add_timeout (
@@ -1462,7 +1438,7 @@
 				  job_name (job), process_name (process), pid);
 		}
 
-		job_process_terminated (job, process, status);
+		job_process_terminated (job, process, status, FALSE);
 		break;
 	case NIH_CHILD_KILLED:
 	case NIH_CHILD_DUMPED:
@@ -1483,7 +1459,7 @@
 		}
 
 		status <<= 8;
-		job_process_terminated (job, process, status);
+		job_process_terminated (job, process, status, FALSE);
 		break;
 	case NIH_CHILD_STOPPED:
 		/* Child was stopped by a signal, make sure it was SIGSTOP
@@ -1560,7 +1536,8 @@
  * job_process_terminated:
  * @job: job that changed,
  * @process: specific process,
- * @status: exit status or signal in higher byte.
+ * @status: exit status or signal in higher byte,
+ * @state_only: if TRUE, only update the jobs state (not its goal).
  *
  * This function is called whenever a @process attached to @job terminates,
  * @status should contain the exit status in the lower byte or signal in
@@ -1572,7 +1549,8 @@
 static void
 job_process_terminated (Job         *job,
 			ProcessType  process,
-			int          status)
+			int          status,
+			int          state_only)
 {
 	int failed = FALSE, stop = FALSE, state = TRUE;
 	struct utmpx *utmptr;
@@ -1580,6 +1558,36 @@
 
 	nih_assert (job != NULL);
 
+	if (job->state == JOB_SECURITY_SPAWNING ||
+			job->state == JOB_PRE_STARTING ||
+			job->state == JOB_SPAWNING ||
+			job->state == JOB_POST_STARTING ||
+			job->state == JOB_PRE_STOPPING ||
+			job->state == JOB_POST_STOPPING) {
+		/* A child process has been spawned by
+		 * job_process_spawn(), but we have not yet received
+		 * confirmation of whether the child setup phase was
+		 * successful or not.
+		 *
+		 * The fact that we are in this function means the child
+		 * setup actually failed, however we handle that
+		 * by waiting for the handlers registered by
+		 * job_register_child_handler() to be called to deal
+		 * with this error.
+		 */
+		nih_assert (job->process_data);
+		nih_assert (job->process_data[process]);
+
+		/* Add the child data for the *second* call to this
+		 * function when the child pipe has been closed.
+		 */
+		job->process_data[process]->status = status;
+
+		job->state = job_next_state (job);
+
+		return;
+	}
+
 	switch (process) {
 	case PROCESS_MAIN:
 		nih_assert ((job->state == JOB_RUNNING)
@@ -1701,7 +1709,7 @@
 		}
 		break;
 	case PROCESS_POST_START:
-		nih_assert (job->state == JOB_POST_START);
+		nih_assert (job->state == JOB_POST_START || job->state == JOB_RUNNING);
 
 		/* We always want to change the state when the post-start
 		 * script terminates; if the main process is running, we'll
@@ -1750,7 +1758,7 @@
 		job->kill_process = PROCESS_INVALID;
 	}
 
-	if (job->class->console == CONSOLE_LOG && job->log[process]) {
+	if (job->class->console == CONSOLE_LOG && job->log[process] && ! state_only) {
 		int  ret;
 
 		/* It is imperative that we free the log at this stage to ensure
@@ -1776,40 +1784,49 @@
 		job->log[process] = NULL;
 	}
 
-	/* Find existing utmp entry for the process pid */
-	setutxent();
-	while ((utmptr = getutxent()) != NULL) {
-		if (utmptr->ut_pid == job->pid[process]) {
-			/* set type and clean ut_user, ut_host,
-			 * ut_time as described in utmp(5)
-			 */
-			utmptr->ut_type = DEAD_PROCESS;
-			memset(utmptr->ut_user, 0, UT_NAMESIZE);
-			memset(utmptr->ut_host, 0, UT_HOSTSIZE);
-			utmptr->ut_time = 0;
-			/* Update existing utmp file. */
-			pututxline(utmptr);
-
-			/* set ut_time for log */
-			gettimeofday(&tv, NULL);
-			utmptr->ut_tv.tv_sec = tv.tv_sec;
-			utmptr->ut_tv.tv_usec = tv.tv_usec;
-			/* Write wtmp entry */
-			updwtmpx (_PATH_WTMP, utmptr);
-
-			break;
+	if (! state_only) {
+		/* Find existing utmp entry for the process pid */
+		setutxent();
+		while ((utmptr = getutxent()) != NULL) {
+			if (utmptr->ut_pid == job->pid[process]) {
+				/* set type and clean ut_user, ut_host,
+				 * ut_time as described in utmp(5)
+				 */
+				utmptr->ut_type = DEAD_PROCESS;
+				memset(utmptr->ut_user, 0, UT_NAMESIZE);
+				memset(utmptr->ut_host, 0, UT_HOSTSIZE);
+				utmptr->ut_time = 0;
+				/* Update existing utmp file. */
+				pututxline(utmptr);
+
+				/* set ut_time for log */
+				gettimeofday(&tv, NULL);
+				utmptr->ut_tv.tv_sec = tv.tv_sec;
+				utmptr->ut_tv.tv_usec = tv.tv_usec;
+				/* Write wtmp entry */
+				updwtmpx (_PATH_WTMP, utmptr);
+
+				break;
+			}
 		}
+		endutxent();
+
+		/* Clear the process pid field */
+		job->pid[process] = 0;
 	}
-	endutxent();
-
-	/* Clear the process pid field */
-	job->pid[process] = 0;
-
 
 	/* Mark the job as failed */
 	if (failed)
 		job_failed (job, process, status);
 
+	/* Cancel goal transition if the job state only should be
+	 * changed.
+	 */
+	if (state_only && stop) {
+		stop = 0;
+		nih_assert (! failed);
+	}
+
 	/* Change the goal to stop; normally this doesn't have any
 	 * side-effects, except when we're in the RUNNING state when it'll
 	 * change the state as well.  We obviously don't want to change the
@@ -1887,7 +1904,7 @@
 	/* Any process can stop on a signal, but we only care about the
 	 * main process when the state is still spawned.
 	 */
-	if ((process != PROCESS_MAIN) || (job->state != JOB_SPAWNED))
+	if ((process != PROCESS_MAIN) || ((job->state != JOB_SPAWNING) && job->state != JOB_SPAWNED))
 		return;
 
 	/* Send SIGCONT back and change the state to the next one, if this
@@ -1916,14 +1933,16 @@
 		       ProcessType  process)
 {
 	nih_assert (job != NULL);
+
 	nih_assert ((job->trace_state == TRACE_NEW)
 		    || (job->trace_state == TRACE_NEW_CHILD));
 
 	/* Any process can get us to trace them, but we only care about the
 	 * main process when the state is still spawned.
 	 */
-	if ((process != PROCESS_MAIN) || (job->state != JOB_SPAWNED))
+	if ((process != PROCESS_MAIN) || ((job->state != JOB_SPAWNING) && (job->state != JOB_SPAWNED))) {
 		return;
+	}
 
 	/* Set options so that we are notified when the process forks, and
 	 * get a different kind of notification when it execs to a plain
@@ -1973,7 +1992,7 @@
 	/* Any process can get us to trace them, but we only care about the
 	 * main process when the state is still spawned.
 	 */
-	if ((process != PROCESS_MAIN) || (job->state != JOB_SPAWNED))
+	if ((process != PROCESS_MAIN) || ((job->state != JOB_SPAWNING) && (job->state != JOB_SPAWNED)))
 		return;
 
 	/* We need to fork at least twice unless we're expecting a
@@ -2018,7 +2037,7 @@
 	/* Any process can get us to trace them, but we only care about the
 	 * main process when the state is still spawned.
 	 */
-	if ((process != PROCESS_MAIN) || (job->state != JOB_SPAWNED)
+	if ((process != PROCESS_MAIN) || ((job->state != JOB_SPAWNING) && (job->state != JOB_SPAWNED))
 	    || (job->trace_state != TRACE_NORMAL))
 		return;
 
@@ -2053,9 +2072,10 @@
 	/* Any process can get us to trace them, but we only care about the
 	 * main process when the state is still spawned.
 	 */
-	if ((process != PROCESS_MAIN) || (job->state != JOB_SPAWNED)
-	    || (job->trace_state != TRACE_NORMAL))
+	if ((process != PROCESS_MAIN) || ((job->state != JOB_SPAWNING) && (job->state != JOB_SPAWNED))
+	    || (job->trace_state != TRACE_NORMAL)) {
 		return;
+	}
 
 	/* Obtain the child process id from the ptrace event. */
 	if (ptrace (PTRACE_GETEVENTMSG, job->pid[process], NULL, &data) < 0) {
@@ -2121,7 +2141,7 @@
 	/* Any process can get us to trace them, but we only care about the
 	 * main process when the state is still spawned and we're tracing it.
 	 */
-	if ((process != PROCESS_MAIN) || (job->state != JOB_SPAWNED)
+	if ((process != PROCESS_MAIN) || ((job->state != JOB_SPAWNING) && job->state != JOB_SPAWNED)
 	    || (job->trace_state != TRACE_NORMAL))
 		return;
 
@@ -2176,7 +2196,6 @@
 				if (job->pid[i] == pid) {
 					if (process)
 						*process = i;
-
 					return job;
 				}
 			}
@@ -2315,16 +2334,426 @@
 
 	nih_assert (fd);
 	nih_assert (reserved_fd);
-	nih_assert (error_fd);
+	nih_assert (error_fd >= 0);
 
-	if (*fd != reserved_fd)
+	if (*fd != reserved_fd) {
+		/* fd does not need remapping */
 		return;
+	}
 
 	new = dup (*fd);
 	if (new < 0) {
 		nih_error_raise_system ();
 		job_process_error_abort (error_fd, JOB_PROCESS_ERROR_DUP, 0);
 	}
+
 	close (*fd);
 	*fd = new;
 }
+
+
+/**
+ * job_process_child_reader:
+ *
+ * @process_data: JobProcessHandlerData structure,
+ * @io: NihIo with data to be read,
+ * @buf: buffer data is available in,
+ * @len: bytes in @buf.
+ *
+ * Called automatically when an error occurred setting up the child process.
+ **/
+void
+job_process_child_reader (JobProcessData  *process_data,
+			  NihIo           *io,
+			  const char      *buf,
+			  size_t           len)
+{
+	Job                     *job;
+	ProcessType              process;
+	NihError                *err;
+
+	/* The job is in the process of being killed which must mean all
+	 * the job pids are dead. Hence, there is no point in attempting
+	 * to talk to them.
+	 */
+	if (! process_data)
+		return;
+	if (! process_data->valid)
+		return;
+
+	nih_assert (io);
+	nih_assert (buf);
+
+	job = process_data->job;
+	process = process_data->process;
+
+	/* Construct the NIH error from the data received from
+	 * the child.
+	 */
+	job_process_error_handler (buf, len);
+
+	err = nih_error_get ();
+
+	nih_assert (err->number == JOB_PROCESS_ERROR);
+
+	/* Non-temporary error condition */
+	nih_warn (_("Failed to spawn %s %s process: %s"),
+			job_name (job), process_name (process),
+			err->message);
+	nih_free (err);
+
+	/* Non-temporary error condition, we're not going
+	 * to be able to spawn this job.
+	 */
+	if (process_data->shell_fd != -1) {
+		close (process_data->shell_fd);
+
+		/* Invalidate */
+		process_data->shell_fd = -1;
+	}
+
+	if (job && job->class->console == CONSOLE_LOG && job->log[process]) {
+		/* Ensure the pty_master watch gets
+		 * removed and the fd closed.
+		 */
+		nih_free (job->log[process]);
+		job->log[process] = NULL;
+	}
+
+	/* Now call the handler registered via job_process_start()
+	 * to deal with the error condition.
+	 */
+	job_child_error_handler (job, process);
+
+	/* Note that pts_master is closed automatically in the parent
+	 * when the log object is destroyed.
+	 */
+
+	nih_io_shutdown (io);
+
+	/* Invalidate */
+	process_data->job_process_fd = -1;
+	process_data->valid = FALSE;
+}
+
+/**
+ * @process_data: JobProcessData,
+ * @io: NihIo.
+ *
+ * NihIoCloseHandler called when job process starts successfully.
+ **/
+void
+job_process_close_handler (JobProcessData  *process_data,
+			   NihIo           *io)
+{
+	Job         *job;
+	ProcessType  process;
+	int          status;
+
+	/* The job is in the process of being killed which must mean all
+	 * the job pids are dead. Hence, there is no point in attempting
+	 * to talk to them.
+	 */
+	if (! process_data)
+		return;
+
+	nih_assert (io);
+
+	job = process_data->job;
+	process = process_data->process;
+	status = process_data->status;
+
+	/* Ensure the job process error fd is closed before attempting
+	 * to handle any scripts.
+	 */
+	nih_free (io);
+
+	/* invalidate */
+	process_data->job_process_fd = -1;
+	process_data->valid = FALSE;
+
+	job_process_run_bottom (process_data);
+
+	if (job && job->state == JOB_SPAWNED) {
+		if (job->class->expect == EXPECT_NONE) {
+			nih_assert (process == PROCESS_MAIN);
+			job_change_state (job, job_next_state (job));
+		}
+	}
+       
+	/* Don't change the jobs goal yet as the process may not have
+	 * actually terminted (and hence will have
+	 * job_process_terminated() called on it again later).
+	 */
+	switch (job->state) {
+		/* FIXME: BUG: incomplete list of states!!!! */
+	case JOB_SECURITY_SPAWNING:
+	case JOB_PRE_STARTING:
+	case JOB_SPAWNING:
+	case JOB_POST_STARTING:
+	case JOB_PRE_STOPPING:
+	case JOB_POST_STOPPING:
+		job_process_terminated (job, process, status, TRUE);
+		break;
+
+	default:
+		/* NOP */
+		break;
+	}
+}
+
+
+/**
+ * job_process_run_bottom:
+ *
+ * @process_data: JobProcessData.
+ *
+ * Perform final job process setup.
+ **/
+void
+job_process_run_bottom (JobProcessData *process_data)
+{
+	Job          *job;
+	//JobState      state;
+	char         *script;
+	int           shell_fd;
+
+	nih_assert (process_data);
+
+	job = process_data->job;
+	shell_fd = process_data->shell_fd;
+	script = process_data->script;
+	//state = process_data->state;
+
+	/* Handle job process containing a script section by sending the
+	 * script to the child shell.
+	 */
+	if (script && shell_fd != -1) {
+		NihIo *io;
+
+
+		/* Put the entire script into an NihIo send buffer and
+		 * then mark it for closure so that the shell gets EOF
+		 * and the structure gets cleaned up automatically.
+		 */
+		while (! (io = nih_io_reopen (job, shell_fd, NIH_IO_STREAM,
+					      NULL, NULL, NULL, NULL))) {
+			NihError *err;
+
+			err = nih_error_get ();
+
+			if (err->number != ENOMEM)
+				nih_assert_not_reached ();
+			nih_free (err);
+		}
+
+		/* We're feeding using a pipe, which has a file descriptor
+		 * on the child end even though it open()s it again using
+		 * a path. Instruct the shell to close this extra fd and
+		 * not to leak it.
+		 */
+		NIH_ZERO (nih_io_printf (io, "exec %d<&-\n",
+					 JOB_PROCESS_SCRIPT_FD));
+
+		NIH_ZERO (nih_io_write (io, script, strlen (script)));
+
+		/* We're only using the io for writing, so clear the
+		 * default flag.
+		 */
+		io->watch->events &= ~NIH_IO_READ;
+
+		nih_io_shutdown (io);
+
+		/* Invalidate now that we've sent the script to the
+		 * child.
+		 */
+		process_data->shell_fd = -1;
+		process_data->valid = FALSE;
+	}
+
+	/* Success, so change the job state */
+	//job->state = state;
+	//job->state = job_next_state (job);
+
+	NIH_LIST_FOREACH (control_conns, iter) {
+		NihListEntry   *entry = (NihListEntry *)iter;
+		DBusConnection *conn = (DBusConnection *)entry->data;
+
+		NIH_ZERO (job_emit_state_changed (
+					conn, job->path,
+					job_state_name (job->state)));
+	}
+}
+
+
+/**
+ * job_process_data_new:
+ *
+ * @parent: parent pointer,
+ * @job: job,
+ * @process: currently-running job process,
+ * @job_process_fd: error file descriptor connected to job process used
+ * to signal successful job process setup.
+ *
+ * Create a new job process data object to record meta-data for an
+ * asynchronously-started job process. The lifetime of such objects is
+ * limited to the time taken for the job process to fail or indicate
+ * successful startup (both scenarios are handled by @job_process_fd).
+ *
+ * Returns: Newly-allocated JobProcess on success, or NULL on
+ * insufficient memory.
+ **/
+JobProcessData *
+job_process_data_new (void         *parent,
+		      Job          *job,
+		      ProcessType   process,
+		      int           job_process_fd)
+{
+	JobProcessData *process_data;
+
+	nih_assert (job);
+	nih_assert (job_process_fd != -1);
+
+	process_data = NIH_MUST (nih_new (parent, JobProcessData));
+	if (! process_data)
+		return NULL;
+
+	process_data->process = process;
+	process_data->job_process_fd = job_process_fd;
+
+	process_data->script = NULL;
+	process_data->shell_fd = -1;
+	process_data->valid = TRUE;
+
+	process_data->job = job;
+	process_data->status = 0;
+
+	return process_data;
+}
+
+/**
+ * job_process_data_serialise:
+ *
+ * @job: job,
+ * @process_data: job process data.
+ *
+ * Returns: JSON-serialised JobProcessData object, or NULL on error.
+ **/
+json_object *
+job_process_data_serialise (const Job *job, const JobProcessData *process_data)
+{
+	json_object     *json;
+
+	nih_assert (job);
+
+	json = json_object_new_object ();
+	if (! json)
+		return NULL;
+
+	/* Job has no "in-flight" process */
+	if (! process_data)
+		return json;
+
+	/* XXX: Note that Job is not serialised; it's not necessary
+	 * since the JobProcessData itself is serialised within its Job
+	 * and the job pointer can be reinstated on deserialisation.
+	 */
+
+	if (! state_set_json_enum_var (json,
+				process_type_enum_to_str,
+				"process", process_data->process))
+		goto error;
+
+	if (! state_set_json_string_var_from_obj (json, process_data, script))
+		goto error;
+
+	if (process_data->shell_fd != -1 && process_data->valid) {
+
+		/* Clear the cloexec flag to ensure the descriptors
+		 * remains open across the re-exec.
+		 */
+		if (state_modify_cloexec (process_data->shell_fd, FALSE) < 0)
+			goto error;
+
+		if (state_modify_cloexec (process_data->job_process_fd, FALSE) < 0)
+			goto error;
+	}
+
+	if (! state_set_json_int_var_from_obj (json, process_data, shell_fd))
+		goto error;
+
+	if (! state_set_json_int_var_from_obj (json, process_data, valid))
+		goto error;
+
+	if (! state_set_json_int_var_from_obj (json, process_data, job_process_fd))
+		goto error;
+
+	return json;
+
+error:
+	json_object_put (json);
+	return NULL;
+}
+
+/**
+ * job_process_data_deserialise:
+ *
+ * @parent: parent pointer,
+ * @job: job,
+ * @json: JSON-serialised JobProcessData object to deserialise.
+ *
+ * Returns: JobProcessData object, or NULL on error.
+ **/
+JobProcessData *
+job_process_data_deserialise (void *parent, Job *job, json_object *json)
+{
+	JobProcessData  *process_data = NULL;
+	ProcessType      process;
+	int              job_process_fd;
+
+	nih_assert (job);
+	nih_assert (json);
+
+	if (! state_check_json_type (json, object))
+		return NULL;
+
+	if (! state_get_json_enum_var (json,
+				process_type_str_to_enum,
+				"process", process))
+		return NULL;
+
+	if (! state_get_json_int_var (json, "job_process_fd", job_process_fd))
+		return NULL;
+
+	process_data = job_process_data_new (parent, job, process, job_process_fd);
+	if (! process_data)
+		return NULL;
+
+	if (! state_get_json_string_var_to_obj (json, process_data, script))
+		goto error;
+
+	if (! state_get_json_int_var_to_obj (json, process_data, shell_fd))
+		goto error;
+
+	if (! state_get_json_int_var_to_obj (json, process_data, valid))
+		goto error;
+
+	if (process_data->shell_fd != -1 && process_data->valid) {
+		/* Reset the cloexec flag to ensure the descriptors
+		 * are not leaked to any child processes.
+		 */
+		if (state_modify_cloexec (process_data->shell_fd, TRUE) < 0)
+			goto error;
+
+		if (state_modify_cloexec (process_data->job_process_fd, TRUE) < 0)
+			goto error;
+	}
+
+	return process_data;
+
+error:
+	if (process_data)
+		nih_free (process_data);
+
+	return NULL;
+}
 
=== modified file 'init/job_process.h'
--- init/job_process.h	2013-05-15 13:21:54 +0000
+++ init/job_process.h	2014-05-22 13:31:23 +0000
@@ -125,15 +125,38 @@
 } JobProcessError;
 
 
+/**
+ * JobProcessErrorHandler:
+ *
+ * @job: job,
+ * @state: required state job is attempting to achieve,
+ * @process: job process that failed.
+ *
+ * Function that is called when a job process @process fails to start.
+ **/
+typedef void (*JobProcessErrorHandler) (Job *job, JobState state, ProcessType process);
+
 NIH_BEGIN_EXTERN
 
-int    job_process_run     (Job *job, ProcessType process);
+void   job_process_start      (Job *job, ProcessType process);
+void   job_process_run_bottom (JobProcessData *handler_data);
+
+void   job_process_child_reader (JobProcessData *handler_data, NihIo *io,
+		const char *buf, size_t len);
+
+void   job_process_close_handler (JobProcessData *handler_data, NihIo *io);
 
 pid_t  job_process_spawn   (Job *job, char * const argv[],
 			    char * const *env, int trace, int script_fd,
 			    ProcessType   process)
 	__attribute__ ((warn_unused_result));
 
+pid_t  job_process_spawn_with_fd   (Job *job, char * const argv[],
+			    char * const *env, int trace, int script_fd,
+			    ProcessType process, int *job_process_fd)
+	__attribute__ ((warn_unused_result));
+
+
 void   job_process_kill    (Job *job, ProcessType process);
 
 void   job_process_handler (void *ptr, pid_t pid,
@@ -145,8 +168,8 @@
 	__attribute__ ((warn_unused_result));
 
 void   job_process_set_kill_timer (Job          *job,
-		  	    ProcessType   process,
-			    time_t        timeout);
+				   ProcessType   process,
+				   time_t        timeout);
 
 void   job_process_adj_kill_timer  (Job *job, time_t due);
 
@@ -154,6 +177,20 @@
 
 void   job_process_stop_all (void);
 
+JobProcessData *
+job_process_data_new (void *parent, Job *job, ProcessType process, int job_process_fd)
+	__attribute__ ((warn_unused_result));
+
+json_object *
+job_process_data_serialise (const Job *job, const JobProcessData *handler_data)
+	__attribute__ ((warn_unused_result));
+
+JobProcessData *
+job_process_data_deserialise (void *parent, Job *job, json_object *json)
+	__attribute__ ((warn_unused_result));
+
+void job_process_error_handler (const char *buf, size_t len);
+
 NIH_END_EXTERN
 
 #endif /* INIT_JOB_PROCESS_H */
 
=== modified file 'init/log.c'
--- init/log.c	2014-05-07 16:34:44 +0000
+++ init/log.c	2014-05-22 13:31:23 +0000
@@ -974,7 +974,7 @@
 	nih_assert (io_watch_fd != -1);
 
 	/* re-apply CLOEXEC flag to stop job fd being leaked to children */
-	if (state_toggle_cloexec (io_watch_fd, TRUE) < 0)
+	if (state_modify_cloexec (io_watch_fd, TRUE) < 0)
 		return NULL;
 
 	if (! state_get_json_int_var (json, "uid", uid))
@@ -993,7 +993,7 @@
 	 * we would never close the fd.
 	 */
 	if (log->fd != -1)
-		(void)state_toggle_cloexec (log->fd, TRUE);
+		(void)state_modify_cloexec (log->fd, TRUE);
 
 	log->unflushed = nih_io_buffer_new (log);
 	if (! log->unflushed)
 
=== modified file 'init/state.c'
--- init/state.c	2014-05-07 16:34:44 +0000
+++ init/state.c	2014-05-22 13:31:23 +0000
@@ -532,7 +532,7 @@
 
 
 /**
- * state_toggle_cloexec:
+ * state_modify_cloexec:
  *
  * @fd: file descriptor,
  * @set: set close-on-exec flag if TRUE, clear if FALSE.
@@ -542,7 +542,7 @@
  * Returns: 0 on success, -1 on error.
  **/
 int
-state_toggle_cloexec (int fd, int set)
+state_modify_cloexec (int fd, int set)
 {
 	long   flags;
 	int    ret;
 
=== modified file 'init/state.h'
--- init/state.h	2013-06-04 16:51:55 +0000
+++ init/state.h	2014-05-22 13:31:23 +0000
@@ -1144,7 +1144,7 @@
 int    state_from_string (const char *state)
 	__attribute__ ((warn_unused_result));
 
-int    state_toggle_cloexec (int fd, int set);
+int    state_modify_cloexec (int fd, int set);
 
 json_object *
 state_serialise_str_array (char ** const array)
 
=== modified file 'init/tests/test_event.c'
--- init/tests/test_event.c	2013-01-31 17:23:55 +0000
+++ init/tests/test_event.c	2014-05-22 13:31:23 +0000
@@ -300,7 +300,7 @@
 		job = (Job *)nih_hash_lookup (class->instances, "");
 
 		TEST_EQ (job->goal, JOB_START);
-		TEST_EQ (job->state, JOB_RUNNING);
+		TEST_EQ (job->state, JOB_SPAWNED);
 		TEST_GT (job->pid[PROCESS_MAIN], 0);
 
 		waitpid (job->pid[PROCESS_MAIN], NULL, 0);
@@ -1739,7 +1739,7 @@
 		job = (Job *)nih_hash_lookup (class->instances, "");
 
 		TEST_EQ (job->goal, JOB_START);
-		TEST_EQ (job->state, JOB_RUNNING);
+		TEST_EQ (job->state, JOB_SPAWNED);
 		TEST_GT (job->pid[PROCESS_MAIN], 0);
 
 		waitpid (job->pid[PROCESS_MAIN], NULL, 0);
 
=== modified file 'init/tests/test_job.c'
--- init/tests/test_job.c	2013-11-18 16:38:32 +0000
+++ init/tests/test_job.c	2014-05-22 13:31:23 +0000
@@ -60,7 +60,15 @@
 #include "conf.h"
 #include "control.h"
 #include "state.h"
+#include "test_util_common.h"
 
+void
+job_quit_with_state (void *data, NihMainLoopFunc *loop)
+{
+    Job *job;
+    job = (Job *)data;
+    nih_main_loop_exit (job->state);
+}
 
 char *argv0;
 
@@ -165,6 +173,12 @@
 			TEST_EQ_P (job->log[i], NULL);
 		}
 
+		TEST_NE_P (job->process_data, NULL);
+		TEST_ALLOC_SIZE (job->log, sizeof (JobProcessData *) * PROCESS_LAST);
+		for (i = 0; i < PROCESS_LAST; i++) {
+			TEST_EQ_P (job->process_data[i], NULL);
+		}
+
 		event_operator_reset (job->stop_on);
 
 		nih_free (job);
@@ -701,6 +715,10 @@
 	char            *path, *job_path = NULL, *state;
 	int              status;
 
+	nih_error_init ();
+	nih_main_loop_init ();
+	event_init ();
+
 	TEST_FUNCTION ("job_change_state");
 	program_name = "test";
 	output = tmpfile ();
@@ -1059,7 +1077,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_PRE_START);
+		job_change_state (job, JOB_PRE_STARTING);
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_PRE_START);
@@ -1110,6 +1128,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1125,7 +1144,8 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_PRE_START);
+		job_change_state (job, JOB_PRE_STARTING);
+		while (nih_main_loop () < JOB_RUNNING) {}
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -1184,6 +1204,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1200,12 +1221,14 @@
 		job->exit_status = 0;
 
 		TEST_DIVERT_STDERR (output) {
-			job_change_state (job, JOB_PRE_START);
+			job_change_state (job, JOB_PRE_STARTING);
+			while (nih_main_loop() < JOB_STOPPING) {}
 		}
 		rewind (output);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_STOPPING);
+		TEST_FALSE (job->process_data[PROCESS_PRE_START]->valid);
 		TEST_EQ (job->pid[PROCESS_PRE_START], 0);
 
 		TEST_EQ (cause->blockers, 0);
@@ -1269,6 +1292,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1284,7 +1308,8 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_SPAWNED);
+		job_change_state (job, JOB_SPAWNING);
+		while (nih_main_loop() < JOB_RUNNING) {}
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -1337,6 +1362,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1352,7 +1378,8 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_SPAWNED);
+		job_change_state (job, JOB_SPAWNING);
+		while (nih_main_loop() < JOB_RUNNING) {}
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -1417,6 +1444,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1432,7 +1460,8 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_SPAWNED);
+		job_change_state (job, JOB_SPAWNING);
+		while (nih_main_loop() < JOB_RUNNING) {}
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -1507,7 +1536,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_SPAWNED);
+		job_change_state (job, JOB_SPAWNING);
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -1556,6 +1585,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1572,12 +1602,14 @@
 		job->exit_status = 0;
 
 		TEST_DIVERT_STDERR (output) {
-			job_change_state (job, JOB_SPAWNED);
+			job_change_state (job, JOB_SPAWNING);
+			while (nih_main_loop() < JOB_STOPPING) {}
 		}
 		rewind (output);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_STOPPING);
+		TEST_FALSE (job->process_data[PROCESS_MAIN]->valid);
 		TEST_EQ (job->pid[PROCESS_MAIN], 0);
 
 		TEST_EQ (cause->blockers, 0);
@@ -1658,7 +1690,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_SPAWNED);
+		job_change_state (job, JOB_SPAWNING);
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_SPAWNED);
@@ -1727,7 +1759,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_POST_START);
+		job_change_state (job, JOB_POST_STARTING);
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_POST_START);
@@ -1794,7 +1826,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_POST_START);
+		job_change_state (job, JOB_POST_STARTING);
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -1842,6 +1874,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_START;
@@ -1858,7 +1891,8 @@
 		job->exit_status = 0;
 
 		TEST_DIVERT_STDERR (output) {
-			job_change_state (job, JOB_POST_START);
+			job_change_state (job, JOB_POST_STARTING);
+			while (nih_main_loop () < JOB_RUNNING) { }
 		}
 		rewind (output);
 
@@ -2058,7 +2092,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_PRE_STOP);
+		job_change_state (job, JOB_PRE_STOPPING);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_PRE_STOP);
@@ -2125,7 +2159,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_PRE_STOP);
+		job_change_state (job, JOB_PRE_STOPPING);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_STOPPING);
@@ -2198,7 +2232,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_PRE_STOP);
+		job_change_state (job, JOB_PRE_STOPPING);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_STOPPING);
@@ -2283,7 +2317,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_PRE_STOP);
+		job_change_state (job, JOB_PRE_STOPPING);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_STOPPING);
@@ -2348,6 +2382,7 @@
 			blocked = blocked_new (job, BLOCKED_EVENT, cause);
 			event_block (cause);
 			nih_list_add (&job->blocking, &blocked->entry);
+			TEST_NE_P (nih_main_loop_add_func (job, job_quit_with_state, job), NULL);
 		}
 
 		job->goal = JOB_STOP;
@@ -2364,7 +2399,8 @@
 		job->exit_status = 0;
 
 		TEST_DIVERT_STDERR (output) {
-			job_change_state (job, JOB_PRE_STOP);
+			job_change_state (job, JOB_PRE_STOPPING);
+			while (nih_main_loop () < JOB_STOPPING) {}
 		}
 		rewind (output);
 
@@ -3100,7 +3136,7 @@
 		job->failed_process = PROCESS_INVALID;
 		job->exit_status = 0;
 
-		job_change_state (job, JOB_POST_STOP);
+		job_change_state (job, JOB_POST_STOPPING);
 
 		TEST_EQ (job->goal, JOB_STOP);
 		TEST_EQ (job->state, JOB_POST_STOP);
@@ -3168,7 +3204,7 @@
 
 		TEST_FREE_TAG (job);
 
-		job_change_state (job, JOB_POST_STOP);
+		job_change_state (job, JOB_POST_STOPPING);
 
 		TEST_FREE (job);
 
@@ -3228,7 +3264,8 @@
 		TEST_FREE_TAG (job);
 
 		TEST_DIVERT_STDERR (output) {
-			job_change_state (job, JOB_POST_STOP);
+			job_change_state (job, JOB_POST_STOPPING);
+			TEST_WATCH_LOOP ();
 		}
 		rewind (output);
 
@@ -4000,22 +4037,31 @@
 
 
 	/* Check that the next state if we're starting a starting job is
+	 * sec-spawning.
+	 */
+	TEST_FEATURE ("with starting job and a goal of start");
+	job->goal = JOB_START;
+	job->state = JOB_STARTING;
+
+	TEST_EQ (job_next_state (job), JOB_SECURITY_SPAWNING);
+
+	/* Check that the next state if we're starting a sec-spawning job is
 	 * security.
 	 */
 	TEST_FEATURE ("with starting job and a goal of start");
 	job->goal = JOB_START;
-	job->state = JOB_STARTING;
+	job->state = JOB_SECURITY_SPAWNING;
 
 	TEST_EQ (job_next_state (job), JOB_SECURITY);
 
 	/* Check that the next state if we're starting a security job is
-	 * pre-start.
+	 * pre-starting.
 	 */
 	TEST_FEATURE ("with security job and a goal of start");
 	job->goal = JOB_START;
 	job->state = JOB_SECURITY;
 
-	TEST_EQ (job_next_state (job), JOB_PRE_START);
+	TEST_EQ (job_next_state (job), JOB_PRE_STARTING);
 
 	/* Check that the next state if we're stopping an security job is
 	 * stopping.
@@ -4026,6 +4072,25 @@
 
 	TEST_EQ (job_next_state (job), JOB_STOPPING);
 
+	/* Check that the next state if we're starting a pre-starting job is
+	 * pre-start.
+	 */
+	TEST_FEATURE ("with pre-stating job and a goal of start");
+	job->goal = JOB_START;
+	job->state = JOB_PRE_STARTING;
+
+	TEST_EQ (job_next_state (job), JOB_PRE_START);
+
+	/* Check that the next state if we're stopping a pre-starting job is
+	 * stopping.
+	 */
+	TEST_FEATURE ("with pre-starting job and a goal of stop");
+	job->goal = JOB_STOP;
+	job->state = JOB_PRE_STARTING;
+
+	TEST_EQ (job_next_state (job), JOB_STOPPING);
+
+
 	/* Check that the next state if we're stopping a pre-start job is
 	 * stopping.
 	 */
@@ -4043,6 +4108,15 @@
 	job->goal = JOB_START;
 	job->state = JOB_PRE_START;
 
+	TEST_EQ (job_next_state (job), JOB_SPAWNING);
+
+	/* Check that the next state if we're starting a spawning job is
+	 * spawned.
+	 */
+	TEST_FEATURE ("with pre-start job and a goal of start");
+	job->goal = JOB_START;
+	job->state = JOB_SPAWNING;
+
 	TEST_EQ (job_next_state (job), JOB_SPAWNED);
 
 
@@ -4057,11 +4131,21 @@
 
 
 	/* Check that the next state if we're starting a spawned job is
+	 * post-starting.
+	 */
+	TEST_FEATURE ("with spawned job and a goal of start");
+	job->goal = JOB_START;
+	job->state = JOB_SPAWNED;
+
+	TEST_EQ (job_next_state (job), JOB_POST_STARTING);
+
+
+	/* Check that the next state if we're starting a post-staring job is
 	 * post-start.
 	 */
-	TEST_FEATURE ("with spawned job and a goal of start");
+	TEST_FEATURE ("with post-starting job and a goal of start");
 	job->goal = JOB_START;
-	job->state = JOB_SPAWNED;
+	job->state = JOB_POST_STARTING;
 
 	TEST_EQ (job_next_state (job), JOB_POST_START);
 
@@ -4106,6 +4190,17 @@
 	job->state = JOB_RUNNING;
 	job->pid[PROCESS_MAIN] = 1;
 
+	TEST_EQ (job_next_state (job), JOB_PRE_STOPPING);
+
+	/* Check that the next state if we're stopping a job with pre-stop is
+	 * pre-stop.  This is the "normal" stop process, as called from the
+	 * goal change event.
+	 */
+	TEST_FEATURE ("with pre-stopping job and a goal of stop");
+	job->goal = JOB_STOP;
+	job->state = JOB_PRE_STOPPING;
+	job->pid[PROCESS_MAIN] = 1;
+
 	TEST_EQ (job_next_state (job), JOB_PRE_STOP);
 
 
@@ -4193,6 +4288,16 @@
 	job->goal = JOB_START;
 	job->state = JOB_KILLED;
 
+	TEST_EQ (job_next_state (job), JOB_POST_STOPPING);
+
+
+	/* Check that the next state if we're starting a post-stopping
+	 * job is post-stop.
+	 */
+	TEST_FEATURE ("with post-stopping job and a goal of start");
+	job->goal = JOB_START;
+	job->state = JOB_POST_STOPPING;
+
 	TEST_EQ (job_next_state (job), JOB_POST_STOP);
 
 
@@ -4203,6 +4308,16 @@
 	job->goal = JOB_STOP;
 	job->state = JOB_KILLED;
 
+	TEST_EQ (job_next_state (job), JOB_POST_STOPPING);
+
+
+	/* Check that the next state if we're stopping a post-stopping
+	 * job is post-stop.
+	 */
+	TEST_FEATURE ("with post-stopping job and a goal of stop");
+	job->goal = JOB_STOP;
+	job->state = JOB_POST_STOPPING;
+
 	TEST_EQ (job_next_state (job), JOB_POST_STOP);
 
 
 
=== modified file 'init/tests/test_job_process.c'
--- init/tests/test_job_process.c	2013-11-11 10:13:08 +0000
+++ init/tests/test_job_process.c	2014-05-22 13:31:23 +0000
@@ -1,3 +1,6 @@
+/*
+ * FIXME: test_job_process has been left behind
+ */
 /* upstart
  *
  * test_job_process.c - test suite for init/job_process.c
@@ -147,33 +150,6 @@
 static int get_available_pty_count (void) __attribute__((unused));
 static void close_all_files (void);
 
-/**
- * fd_valid:
- * @fd: file descriptor.
- *
- * Return 1 if @fd is valid, else 0.
- **/
-static int
-fd_valid (int fd)
-{
-	int flags = 0;
-
-	if (fd < 0)
-		return 0;
-
-	errno = 0;
-	flags = fcntl (fd, F_GETFL);
-
-	if (flags < 0)
-		return 0;
-
-	/* redundant really */
-	if (errno == EBADF)
-		return 0;
-
-	return 1;
-}
-
 static void
 child (enum child_tests  test,
        const char       *filename)
@@ -409,7 +385,7 @@
  * (Such tests are handled in the bundled test_user_sessions.sh script).
  */
 void
-test_run (void)
+test_start (void)
 {
 	char             dirname[PATH_MAX];
 	JobClass        *class = NULL;
@@ -418,7 +394,7 @@
 	struct stat      statbuf;
 	char             filename[PATH_MAX], buf[80];
 	char             function[PATH_MAX];
-	int              ret = -1, status, first;
+	int              status;
 	siginfo_t        info;
 	char             filebuf[1024];
 	struct passwd   *pwd;
@@ -433,7 +409,7 @@
 	log_unflushed_init ();
 	job_class_init ();
 
-	TEST_FUNCTION ("job_process_run");
+	TEST_FUNCTION ("job_process_start");
 
 	TEST_FILENAME (filename);
 	program_name = "test";
@@ -467,8 +443,7 @@
 			job->state = JOB_SPAWNED;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -502,8 +477,7 @@
 			job->state = JOB_SPAWNED;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -544,8 +518,7 @@
 			job->state = JOB_SPAWNED;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -584,8 +557,7 @@
 			job->state = JOB_SPAWNED;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -625,8 +597,7 @@
 			job->state = JOB_SPAWNED;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -672,8 +643,7 @@
 						   "CRACKLE=FIZZ"));
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -728,8 +698,7 @@
 						   "CRACKLE=FIZZ"));
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -785,8 +754,7 @@
 						   "CRACKLE=FIZZ"));
 		}
 
-		ret = job_process_run (job, PROCESS_PRE_STOP);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_PRE_STOP);
 
 		TEST_NE (job->pid[PROCESS_PRE_STOP], 0);
 
@@ -843,8 +811,7 @@
 						   "CRACKLE=FIZZ"));
 		}
 
-		ret = job_process_run (job, PROCESS_POST_STOP);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_POST_STOP);
 
 		TEST_NE (job->pid[PROCESS_POST_STOP], 0);
 
@@ -894,36 +861,11 @@
 			job->state = JOB_SPAWNED;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
-		/* Loop until we've fed all of the data. */
-		first = TRUE;
-		for (;;) {
-			fd_set readfds, writefds, exceptfds;
-			int    nfds;
-
-			nfds = 0;
-			FD_ZERO (&readfds);
-			FD_ZERO (&writefds);
-			FD_ZERO (&exceptfds);
-
-			nih_io_select_fds (&nfds, &readfds,
-					   &writefds, &exceptfds);
-			if (! nfds) {
-				if (first)
-					TEST_FAILED ("expected to have "
-						     "data to feed.");
-				break;
-			}
-			first = FALSE;
-
-			select (nfds, &readfds, &writefds, &exceptfds, NULL);
-
-			nih_io_handle_fds (&readfds, &writefds, &exceptfds);
-		}
+		TEST_WATCH_LOOP ();
 
 		waitpid (job->pid[PROCESS_MAIN], &status, 0);
 		TEST_TRUE (WIFEXITED (status));
@@ -962,8 +904,7 @@
 			job->trace_state = TRACE_NORMAL;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_EQ (job->trace_forks, 0);
 		TEST_EQ (job->trace_state, TRACE_NONE);
@@ -1002,8 +943,7 @@
 			job->trace_state = TRACE_NORMAL;
 		}
 
-		ret = job_process_run (job, PROCESS_PRE_START);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_PRE_START);
 
 		TEST_EQ (job->trace_forks, 0);
 		TEST_EQ (job->trace_state, TRACE_NONE);
@@ -1044,8 +984,7 @@
 			job->trace_state = TRACE_NORMAL;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_EQ (job->trace_forks, 0);
 		TEST_EQ (job->trace_state, TRACE_NEW);
@@ -1095,8 +1034,7 @@
 			job->trace_state = TRACE_NORMAL;
 		}
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_EQ (job->trace_forks, 0);
 		TEST_EQ (job->trace_state, TRACE_NEW);
@@ -1122,7 +1060,7 @@
 	}
 
 	/* Check that if we try and run a command that doesn't exist,
-	 * job_process_run() raises a ProcessError and the command doesn't
+	 * job_process_start() raises a ProcessError and the command doesn't
 	 * have any stored process id for it.
 	 */
 	TEST_FEATURE ("with no such file");
@@ -1144,11 +1082,12 @@
 		}
 
 		TEST_DIVERT_STDERR (output) {
-			ret = job_process_run (job, PROCESS_MAIN);
+			job_process_start (job, PROCESS_MAIN);
+			TEST_WATCH_LOOP ();
+			event_poll ();
 		}
 		rewind (output);
-		TEST_LT (ret, 0);
-
+		
 		TEST_EQ (job->pid[PROCESS_MAIN], 0);
 
 		TEST_FILE_EQ (output, ("test: Failed to spawn test (foo) main "
@@ -1195,8 +1134,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1263,8 +1201,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1331,8 +1268,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1399,8 +1335,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1462,8 +1397,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1499,8 +1433,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1536,8 +1469,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1583,11 +1515,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/* XXX: call 0: async process setup */
+	TEST_WATCH_UPDATE ();
+
 	/* XXX: call 1: wait for script write to child shell */
 	TEST_WATCH_UPDATE ();
 
@@ -1613,10 +1547,10 @@
 
 	/* Note we can't use TEST_ALLOC_FAIL() for this test since on
 	 * the ENOMEM loop all we could do is discard the error and
-	 * continue since job_process_run() calls job_process_spawn()
+	 * continue since job_process_start() calls job_process_spawn()
 	 * repeatedly until it works, but the alloc fails in log_new()
 	 * invoked by job_process_spawn() such that when we've left
-	 * job_process_run(), it's too late.
+	 * job_process_start(), it's too late.
 	 *
 	 * However, we test this scenario in test_spawn() so all is not
 	 * lost.
@@ -1637,8 +1571,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -1695,11 +1628,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/*  wait for process to setup */
+	TEST_WATCH_UPDATE ();
+
 	/*  wait for read from pty allowing logger to write to log file */
 	TEST_WATCH_UPDATE ();
 
@@ -1779,11 +1714,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/*  wait for process to setup */
+	TEST_WATCH_UPDATE ();
+
 	/*  wait for read from pty allowing logger to write to log file */
 	TEST_WATCH_UPDATE ();
 
@@ -1861,10 +1798,10 @@
 
 	/* Note we can't use TEST_ALLOC_FAIL() for this test since on
 	 * the ENOMEM loop all we could do is discard the error and
-	 * continue since job_process_run() calls job_process_spawn()
+	 * continue since job_process_start() calls job_process_spawn()
 	 * repeatedly until it works, but the alloc fails in log_new()
 	 * invoked by job_process_spawn() such that when we've left
-	 * job_process_run(), it's too late.
+	 * job_process_start(), it's too late.
 	 *
 	 * However, we test this scenario in test_spawn() so all is not
 	 * lost.
@@ -1885,11 +1822,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/*  wait for process to setup */
+	TEST_WATCH_UPDATE ();
+
 	/* XXX: call 1: wait for script write to child shell */
 	TEST_WATCH_UPDATE ();
 
@@ -1948,11 +1887,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/*  wait for process to setup */
+	TEST_WATCH_UPDATE ();
+
 	/*  wait for read from pty allowing logger to write to log file */
 	TEST_WATCH_UPDATE ();
 
@@ -2017,11 +1958,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/*  wait for process to setup */
+	TEST_WATCH_UPDATE ();
+
 	/* XXX: call 1: wait for script write to child shell */
 	TEST_WATCH_UPDATE ();
 
@@ -2064,10 +2007,10 @@
 
 	/* Note we can't use TEST_ALLOC_FAIL() for this test since on
 	 * the ENOMEM loop all we could do is discard the error and
-	 * continue since job_process_run() calls job_process_spawn()
+	 * continue since job_process_start() calls job_process_spawn()
 	 * repeatedly until it works, but the alloc fails in log_new()
 	 * invoked by job_process_spawn() such that when we've left
-	 * job_process_run(), it's too late.
+	 * job_process_start(), it's too late.
 	 *
 	 * However, we test this scenario in test_spawn() so all is not
 	 * lost.
@@ -2088,8 +2031,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2135,7 +2077,7 @@
 	 * XXX: TEST_WATCH_UPDATE() *TWICE* to ensure select(2) is
 	 * XXX: called twice.
 	 *
-	 * This is required since job_process_run() uses an NihIo object
+	 * This is required since job_process_start() uses an NihIo object
 	 * to squirt the script to the shell sub-process and this
 	 * triggers select to return when the data is written to the shell.
 	 * However, we don't care about that directly - we care more about
@@ -2144,7 +2086,7 @@
 	 * written.
 	 *
 	 * Note that the 2nd call to TEST_WATCH_UPDATE would not be
-	 * required should job_process_run() simple invoke write(2) to
+	 * required should job_process_start() simple invoke write(2) to
 	 * send the data.
 	 */
 
@@ -2165,11 +2107,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
+	/*  wait for process to setup */
+	TEST_WATCH_UPDATE ();
+
 	/* XXX: call 1: wait for script write to child shell */
 	TEST_WATCH_UPDATE ();
 
@@ -2227,8 +2171,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2287,8 +2230,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2352,8 +2294,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2414,12 +2355,12 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
 	TEST_WATCH_UPDATE ();
+	TEST_WATCH_UPDATE ();
 	waitpid (job->pid[PROCESS_MAIN], &status, 0);
 	TEST_TRUE (WIFEXITED (status));
 	TEST_EQ (WEXITSTATUS (status), 0);
@@ -2474,8 +2415,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2537,8 +2477,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2597,12 +2536,12 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
 	TEST_WATCH_UPDATE ();
+	TEST_WATCH_UPDATE ();
 
 	waitpid (job->pid[PROCESS_MAIN], &status, 0);
 	TEST_TRUE (WIFEXITED (status));
@@ -2659,8 +2598,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2725,8 +2663,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2787,12 +2724,12 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
 	TEST_WATCH_UPDATE ();
+	TEST_WATCH_UPDATE ();
 
 	waitpid (job->pid[PROCESS_MAIN], &status, 0);
 	TEST_TRUE (WIFEXITED (status));
@@ -2851,8 +2788,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2913,8 +2849,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -2973,8 +2908,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -3037,8 +2971,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -3112,8 +3045,9 @@
 	output = tmpfile ();
 	TEST_NE_P (output, NULL);
 	TEST_DIVERT_STDERR (output) {
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
+		TEST_WATCH_UPDATE ();
+		event_poll ();
 	}
 	fclose (output);
 
@@ -3161,8 +3095,8 @@
 		job->goal = JOB_START;
 		job->state = JOB_SPAWNED;
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
+		TEST_WATCH_UPDATE ();
 
 		/* We don't expect a logfile to be written since there is no
 		 * accompanying shell to write the error.
@@ -3173,10 +3107,10 @@
 		job->goal = JOB_STOP;
 		job->state = JOB_POST_STOP;
 
-		ret = job_process_run (job, PROCESS_POST_STOP);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_POST_STOP);
 
 		TEST_NE (job->pid[PROCESS_POST_STOP], 0);
+		TEST_WATCH_UPDATE ();
 
 		/* Flush the io so that the shell on the client side
 		 * gets the data (the script to execute).
@@ -3187,8 +3121,11 @@
 		TEST_TRUE (WIFEXITED (status));
 		TEST_EQ (WEXITSTATUS (status), 0);
 
+		TEST_WATCH_UPDATE ();
+
 		/* .. but the post stop should have written data */
 		TEST_EQ (stat (filename, &statbuf), 0);
+		event_poll ();
 	}
 	fclose (output);
 
@@ -3241,8 +3178,9 @@
 		job->goal = JOB_START;
 		job->state = JOB_SPAWNED;
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
+		TEST_WATCH_UPDATE ();
+		TEST_WATCH_UPDATE ();
 
 		/* We don't expect a logfile to be written since there is no
 		 * accompanying shell to write the error.
@@ -3253,8 +3191,8 @@
 		job->goal = JOB_STOP;
 		job->state = JOB_POST_STOP;
 
-		ret = job_process_run (job, PROCESS_POST_STOP);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_POST_STOP);
+		TEST_WATCH_UPDATE ();
 
 		TEST_NE (job->pid[PROCESS_POST_STOP], 0);
 
@@ -3272,6 +3210,7 @@
 
 		/* .. but the post stop should have written data */
 		TEST_EQ (stat (filename, &statbuf), 0);
+		event_poll ();
 	}
 	fclose (output);
 
@@ -3325,8 +3264,8 @@
 		job->goal = JOB_START;
 		job->state = JOB_SPAWNED;
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
+		TEST_WATCH_UPDATE ();
 
 		/* We don't expect a logfile to be written since there is no
 		 * accompanying shell to write the error.
@@ -3337,8 +3276,8 @@
 		job->goal = JOB_STOP;
 		job->state = JOB_POST_STOP;
 
-		ret = job_process_run (job, PROCESS_POST_STOP);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_POST_STOP);
+		TEST_WATCH_UPDATE ();
 
 		TEST_NE (job->pid[PROCESS_POST_STOP], 0);
 
@@ -3351,8 +3290,11 @@
 		TEST_TRUE (WIFEXITED (status));
 		TEST_EQ (WEXITSTATUS (status), 0);
 
+		TEST_WATCH_UPDATE ();
+
 		/* .. but the post stop should have written data */
 		TEST_EQ (stat (filename, &statbuf), 0);
+		event_poll ();
 	}
 	fclose (output);
 
@@ -3405,8 +3347,8 @@
 		job->goal = JOB_START;
 		job->state = JOB_SPAWNED;
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
+		TEST_WATCH_UPDATE ();
 
 		/* We don't expect a logfile to be written since there is no
 		 * accompanying shell to write the error.
@@ -3417,12 +3359,13 @@
 		job->goal = JOB_STOP;
 		job->state = JOB_POST_STOP;
 
-		ret = job_process_run (job, PROCESS_POST_STOP);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_POST_STOP);
+		TEST_WATCH_UPDATE ();
 
 		/* Again, no file expected */
 		TEST_EQ (stat (filename, &statbuf), -1);
 		TEST_EQ (errno, ENOENT);
+		event_poll ();
 	}
 	fclose (output);
 	nih_free (class);
@@ -3463,8 +3406,7 @@
 		job->goal = JOB_START;
 		job->state = JOB_SPAWNED;
 
-		ret = job_process_run (job, PROCESS_MAIN);
-		TEST_EQ (ret, 0);
+		job_process_start (job, PROCESS_MAIN);
 
 		TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -3483,8 +3425,8 @@
 		job->goal = JOB_STOP;
 		job->state = JOB_POST_STOP;
 
-		ret = job_process_run (job, PROCESS_POST_STOP);
-		TEST_LT (ret, 0);
+		job_process_start (job, PROCESS_POST_STOP);
+		TEST_WATCH_UPDATE ();
 
 		TEST_EQ (job->pid[PROCESS_POST_STOP], 0);
 	}
@@ -3524,8 +3466,8 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
+	TEST_WATCH_UPDATE ();
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -3585,8 +3527,8 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
+	TEST_WATCH_UPDATE ();
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -3648,8 +3590,8 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
+	TEST_WATCH_UPDATE ();
 
 	TEST_NE (job->pid[PROCESS_MAIN], 0);
 
@@ -3728,8 +3670,7 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
 
 	/* Wait for process to avoid any possibility of EAGAIN in
 	 * log_read_watch().
@@ -3802,8 +3743,9 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
+	TEST_WATCH_UPDATE ();
+	TEST_WATCH_UPDATE ();
 
 	pid = job->pid[PROCESS_MAIN];
 
@@ -3933,8 +3875,8 @@
 		TEST_EQ_P (job->log[i], NULL);
 	}
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_MAIN);
+	TEST_WATCH_UPDATE ();
 
 	pid = job->pid[PROCESS_MAIN];
 
@@ -4006,8 +3948,7 @@
 		}
 
 		TEST_DIVERT_STDERR (output) {
-			ret = job_process_run (job, PROCESS_MAIN);
-			TEST_EQ (ret, 0);
+			job_process_start (job, PROCESS_MAIN);
 		}
 		fclose (output);
 
@@ -4064,13 +4005,8 @@
 		}
 
 		TEST_DIVERT_STDERR (output) {
-			ret = job_process_run (job, PROCESS_MAIN);
-			if (geteuid() == 0 || getuid() == pwd->pw_uid) {
-				TEST_EQ (ret, 0);
-			}
-			else {
-				TEST_EQ (ret, -1);
-			}
+			job_process_start (job, PROCESS_MAIN);
+			TEST_WATCH_UPDATE ();
 		}
 
 		if (geteuid() == 0 || getuid() == pwd->pw_uid) {
@@ -4081,13 +4017,15 @@
 		}
 		else {
 			TEST_EQ (stat (filename, &statbuf), -1);
+			event_poll ();
 		}
 
 		unlink (filename);
 		nih_free (class);
-
 	}
 
+	/* FIXME with async spawn this test is racy */
+
 	/************************************************************/
 	TEST_FEATURE ("with multiple processes and log");
 	TEST_HASH_EMPTY (job_classes);
@@ -4128,14 +4066,13 @@
 	job->goal = JOB_START;
 	job->state = JOB_SPAWNED;
 
-	ret = job_process_run (job, PROCESS_MAIN);
-	TEST_EQ (ret, 0);
-
+	job_process_start (job, PROCESS_MAIN);
+	while (stat (filename, &statbuf) != 0) {
+		TEST_WATCH_UPDATE ();
+	}
 	pid = job->pid[PROCESS_MAIN];
 	TEST_GT (pid, 0);
 
-	TEST_WATCH_UPDATE ();
-
 	TEST_EQ (stat (filename, &statbuf), 0);
 
 	output = fopen (filename, "r");
@@ -4148,8 +4085,8 @@
 
 	TEST_EQ (fclose (output), 0);
 
-	ret = job_process_run (job, PROCESS_POST_START);
-	TEST_EQ (ret, 0);
+	job_process_start (job, PROCESS_POST_START);
+	TEST_WATCH_UPDATE ();
 
 	pid = job->pid[PROCESS_POST_START];
 	TEST_GT (pid, 0);
@@ -4244,6 +4181,8 @@
 	int               status;
 	struct stat       statbuf;
 	int               ret;
+	int               job_process_fd = -1;
+	nih_local NihIoBuffer *buffer = NULL;
 
 	log_unflushed_init ();
 
@@ -4257,7 +4196,7 @@
 	 */
 	TEST_EQ (setenv ("UPSTART_LOGDIR", dirname, 1), 0);
 
-	TEST_FUNCTION ("job_process_spawn");
+	TEST_FUNCTION ("job_process_spawn_with_fd");
 	TEST_FILENAME (filename);
 
 	args[0] = argv0;
@@ -4278,7 +4217,7 @@
 	class->console = CONSOLE_NONE;
 	job   = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4319,7 +4258,7 @@
 	class->console = CONSOLE_NONE;
 	job = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4351,7 +4290,7 @@
 	class->console = CONSOLE_LOG;
 	job = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4398,7 +4337,7 @@
 	class->chdir = "/tmp";
 	job = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4430,7 +4369,7 @@
 	class->console = CONSOLE_NONE;
 	job   = job_new (class, "");
 
-	pid = job_process_spawn (job, args, env, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, env, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4460,7 +4399,7 @@
 	class->console = CONSOLE_NONE;
 	job   = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	assert0 (waitid (P_PID, pid, &info, WEXITED | WSTOPPED | WCONTINUED));
@@ -4483,7 +4422,7 @@
 	class = job_class_new (NULL, "test", NULL);
 	job   = job_new (class, "");
 	class->console = CONSOLE_NONE;
-	pid = job_process_spawn (job, args, NULL, TRUE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, TRUE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	assert0 (waitid (P_PID, pid, &info, WEXITED | WSTOPPED | WCONTINUED));
@@ -4516,8 +4455,12 @@
 	class->console = CONSOLE_NONE;
 	job   = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
-	TEST_LT (pid, 0);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
+	TEST_NE (pid, 0);
+
+	buffer = read_from_fd (NULL, job_process_fd);
+	TEST_NE_P (buffer, NULL);
+	job_process_error_handler (buffer->buf, buffer->len);
 
 	err = nih_error_get ();
 	TEST_EQ (err->number, JOB_PROCESS_ERROR);
@@ -4528,6 +4471,7 @@
 	TEST_EQ (perr->arg, 0);
 	TEST_EQ (perr->errnum, ENOENT);
 	nih_free (perr);
+	nih_free (buffer);
 
 	/************************************************************/
 	TEST_FEATURE ("with no such file, no shell and console log");
@@ -4539,29 +4483,36 @@
 	class = job_class_new (NULL, "test", NULL);
 	class->console = CONSOLE_LOG;
 	job   = job_new (class, "");
+	job->goal = JOB_START;
+	job->state = JOB_SPAWNED;
 
 	TEST_NE_P (job->log, NULL);
 	TEST_EQ_P (job->log[PROCESS_MAIN], NULL);
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
-	TEST_LT (pid, 0);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
+	job->process_data[PROCESS_MAIN] = NIH_MUST (
+		job_process_data_new (job->process_data, job, PROCESS_MAIN, job_process_fd));
+	TEST_WATCH_UPDATE ();
+	TEST_NE (pid, 0);
 
 	TEST_GT (waitpid (-1, NULL, 0), 0);
 
+	NihIo *io = nih_io_reopen (job, job_process_fd, NIH_IO_STREAM, NULL, NULL, NULL, NULL);
+	TEST_NE_P (io, NULL);
+	buffer = read_from_fd (NULL, job_process_fd);
+	TEST_NE_P (buffer, NULL);
+	TEST_NE_P (job->process_data[PROCESS_MAIN], NULL);
+	TEST_TRUE (job->process_data[PROCESS_MAIN]->valid);
+	
+
+	TEST_NE_P (job->log[PROCESS_MAIN], NULL);
+
+	job_process_child_reader (job->process_data[PROCESS_MAIN], io, buffer->buf, buffer->len);
+
 	/* The log should have been allocated in job_process_spawn,
 	 * but then freed on error.
 	 */
 	TEST_EQ_P (job->log[PROCESS_MAIN], NULL);
 
-	err = nih_error_get ();
-	TEST_EQ (err->number, JOB_PROCESS_ERROR);
-	TEST_ALLOC_SIZE (err, sizeof (JobProcessError));
-
-	perr = (JobProcessError *)err;
-	TEST_EQ (perr->type, JOB_PROCESS_ERROR_EXEC);
-	TEST_EQ (perr->arg, 0);
-	TEST_EQ (perr->errnum, ENOENT);
-	nih_free (perr);
-
 	/* Check that we can spawn a job and pause it
 	 */
 	TEST_FEATURE ("with debug enabled");
@@ -4577,7 +4528,7 @@
 	args[1] = function;
 	args[2] = NULL;
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	/* Ensure process is still running after some period of time.
@@ -4614,7 +4565,7 @@
 	class->console = CONSOLE_NONE;
 	job = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4669,7 +4620,7 @@
 	class->console = CONSOLE_LOG;
 	job = job_new (class, "");
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	waitpid (pid, NULL, 0);
@@ -4742,8 +4693,8 @@
 	args[2] = filebuf;
 	args[3] = NULL;
 
-	job->pid[PROCESS_MAIN] = job_process_spawn (job, args, NULL,
-			FALSE, -1, PROCESS_MAIN);
+	job->pid[PROCESS_MAIN] = job_process_spawn_with_fd (job, args, NULL,
+							    FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	pid = job->pid[PROCESS_MAIN];
 	TEST_GT (pid, 0);
 
@@ -4769,8 +4720,8 @@
 	args[2] = filebuf;
 	args[3] = NULL;
 
-	job->pid[PROCESS_POST_START] = job_process_spawn (job, args, NULL,
-			FALSE, -1, PROCESS_POST_START);
+	job->pid[PROCESS_POST_START] = job_process_spawn_with_fd (job, args, NULL,
+								  FALSE, -1, PROCESS_POST_START, &job_process_fd);
 	pid = job->pid[PROCESS_POST_START];
 	TEST_GT (pid, 0);
 
@@ -4867,7 +4818,7 @@
 			NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, script));
 		}
 
-		pid = job_process_spawn (job, args_array, NULL, FALSE, -1, PROCESS_MAIN);
+		pid = job_process_spawn_with_fd (job, args_array, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 
 		if (test_alloc_failed) {
 			TEST_LT (pid, 0);
@@ -4934,7 +4885,7 @@
 	NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, TEST_SHELL_ARG));
 	NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, script));
 
-	pid = job_process_spawn (job, args_array, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args_array, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	TEST_EQ (waitpid (pid, &status, 0), pid);
@@ -4990,7 +4941,7 @@
 	NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, TEST_SHELL_ARG));
 	NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, script));
 
-	pid = job_process_spawn (job, args_array, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args_array, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	TEST_EQ (waitpid (pid, &status, 0), pid);
@@ -5043,7 +4994,7 @@
 	NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, "-en"));
 	NIH_MUST (nih_str_array_add (&args_array, NULL, &argc, "\\000"));
 
-	pid = job_process_spawn (job, args_array, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args_array, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	TEST_EQ (waitpid (pid, &status, 0), pid);
@@ -5098,7 +5049,7 @@
 	args[3] = filebuf;
 	args[4] = NULL;
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	TEST_NE (waitpid (pid, &status, 0), -1);
@@ -5161,7 +5112,7 @@
 	args[3] = filebuf;
 	args[4] = NULL;
 
-	pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
+	pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
 	TEST_GT (pid, 0);
 
 	TEST_WATCH_UPDATE ();
@@ -5243,8 +5194,13 @@
 				}
 			}
 
-			pid = job_process_spawn (job, args, NULL, FALSE, -1, PROCESS_MAIN);
-			TEST_LT (pid, 0);
+			pid = job_process_spawn_with_fd (job, args, NULL, FALSE, -1, PROCESS_MAIN, &job_process_fd);
+			TEST_WATCH_UPDATE ();
+			TEST_NE (pid, 0);
+
+			buffer = read_from_fd (NULL, job_process_fd);
+			TEST_NE_P (buffer, NULL);
+			job_process_error_handler (buffer->buf, buffer->len);
 
 			/* Ensure logging disabled in failure scenarios */
 			TEST_EQ (class->console, CONSOLE_NONE);
@@ -5830,6 +5786,8 @@
 		job->exit_status = 0;
 
 		job_process_handler (NULL, 1, NIH_CHILD_EXITED, 0);
+		TEST_EQ (job->state, JOB_SPAWNED);
+		job_change_state (job, job_next_state(job));
 
 		TEST_EQ (job->goal, JOB_START);
 		TEST_EQ (job->state, JOB_RUNNING);
@@ -9181,7 +9139,7 @@
 void
 run_tests (void)
 {
-	test_run ();
+	test_start ();
 	test_spawn ();
 	test_log_path ();
 	test_kill ();
 
=== modified file 'init/tests/test_util.c'
--- init/tests/test_util.c	2013-07-17 14:18:42 +0000
+++ init/tests/test_util.c	2014-05-22 13:31:23 +0000
@@ -1,5 +1,6 @@
+#include "test_util_common.h"
+#undef PACKAGE_COPYRIGHT
 #include "test_util.h"
-#include "test_util_common.h"
 #include <nih/logging.h>
 #include <nih/test.h>
 
 
=== modified file 'test/test_util_common.c'
--- test/test_util_common.c	2014-04-11 21:15:39 +0000
+++ test/test_util_common.c	2014-05-22 13:31:23 +0000
@@ -28,6 +28,10 @@
 #include <nih/string.h>
 #include <nih/signal.h>
 #include <nih/logging.h>
+#include <nih/timer.h>
+#include <nih/io.h>
+#include <nih/child.h>
+#include <nih/main.h>
 #include <nih-dbus/test_dbus.h>
 
 #include <dbus/dbus.h>
@@ -986,7 +990,7 @@
 
 			/* Clean up if tests forgot to */
 			cmd = NIH_MUST (nih_sprintf (NULL, "rm %s/*.session 2>/dev/null", path));
-			system (cmd);
+			assert0 (system (cmd));
 
 			/* Remove the directory tree the first Session Init created */
 			assert0 (rmdir (path));
@@ -1003,3 +1007,89 @@
 
 	}
 }
+
+/**
+ * timer_cb:
+ *
+ * @data: unused,
+ * @timer: timer.
+ *
+ * Exit main loop with an error value indicating that the expected main
+ * loop events/actions were not performed within the expected time.
+ **/
+void
+timer_cb (void *data, NihTimer *timer)
+{
+	nih_assert (timer);
+
+	/* Return non-zero to denote failure */
+	nih_main_loop_exit (42);
+}
+
+/**
+ * fd_valid:
+ * @fd: file descriptor.
+ *
+ * Return 1 if @fd is valid, else 0.
+ **/
+int
+fd_valid (int fd)
+{
+	int flags = 0;
+
+	if (fd < 0)
+		return 0;
+
+	errno = 0;
+	flags = fcntl (fd, F_GETFL);
+
+	if (flags < 0)
+		return 0;
+
+	/* redundant really */
+	if (errno == EBADF)
+		return 0;
+
+	return 1;
+}
+
+/**
+ * read_from_fd:
+ *
+ * @parent: parent,
+ * @fd: open file descriptor.
+ *
+ * Read from the specified fd, close the fd and return the data.
+ *
+ * Returns: Newly-allocated NihIoBuffer representing data read from @fd.
+ *
+ **/
+NihIoBuffer *
+read_from_fd (void *parent, int fd)
+{
+	NihIoBuffer *buffer = NULL;
+	ssize_t      len;
+
+	assert (fd >= 0);
+
+	buffer = nih_io_buffer_new (parent);
+	nih_assert (buffer);
+
+	while (TRUE) {
+
+		nih_assert (! nih_io_buffer_resize (buffer, 1024));
+
+		len = read (fd,
+				buffer->buf + buffer->len,
+				buffer->size - buffer->len);
+
+		if (len <= 0)
+			break;
+		else if (len > 0)
+			buffer->len += len;
+	}
+
+	close (fd);
+
+	return buffer;
+}
 
=== modified file 'test/test_util_common.h'
--- test/test_util_common.h	2013-11-15 23:47:31 +0000
+++ test/test_util_common.h	2014-05-22 13:31:23 +0000
@@ -6,6 +6,10 @@
 #include <sys/wait.h>
 
 #include <nih-dbus/test_dbus.h>
+#include <nih/timer.h>
+#include <nih/main.h>
+#include <nih/child.h>
+#include <nih/io.h>
 
 /**
  * TEST_DIR_MODE:
@@ -34,6 +38,16 @@
 
 #define TEST_QUIESCE_TOTAL_WAIT_TIME (TEST_EXIT_TIME + TEST_QUIESCE_KILL_PHASE)
 
+/**
+ * TEST_MAIN_LOOP_TIMEOUT_SECS:
+ *
+ * Number of seconds to wait until the main loop is exited in error.
+ *
+ * To avoid a test failure, all main loops must exit within this
+ * number of seconds.
+ **/
+#define TEST_MAIN_LOOP_TIMEOUT_SECS    5
+
 /* A 'reasonable' path, but which also contains a marker at the end so
  * we know when we're looking at a PATH these tests have set.
  */
@@ -74,6 +88,41 @@
 }
 
 /**
+ * TEST_WATCH_LOOP:
+ * 
+ * Loop for NihIo object Updates, and process them until no watches
+ * left.
+ */
+#define TEST_WATCH_LOOP()                                            \
+{							             \
+	/* Loop until we've fed all of the data. */                  \
+	int first = TRUE;                                            \
+	for (;;) {                                                   \
+		fd_set readfds, writefds, exceptfds;                 \
+		int    nfds;					     \
+								     \
+		nfds = 0;					     \
+		FD_ZERO (&readfds);				     \
+		FD_ZERO (&writefds);				     \
+		FD_ZERO (&exceptfds);				     \
+								     \
+		nih_io_select_fds (&nfds, &readfds,		     \
+				   &writefds, &exceptfds);	     \
+		if (! nfds) {					     \
+		    if (first)					     \
+			TEST_FAILED ("expected to have "	     \
+				     "data to feed.");		     \
+		    break;					     \
+		}						     \
+		first = FALSE;					     \
+									\
+		select (nfds, &readfds, &writefds, &exceptfds, NULL);	\
+								     \
+		nih_io_handle_fds (&readfds, &writefds, &exceptfds); \
+	}							     \
+}
+
+/**
  * _TEST_WATCH_UPDATE:
  * @force: if TRUE, force an update,
  * @timeout: struct timeval pointer, or NULL if no timeout required.
@@ -169,6 +218,30 @@
 }
 
 /**
+ * TEST_RESET_MAIN_LOOP:
+ * 
+ * Reset main loop and associated test variables.
+ **/
+#define TEST_RESET_MAIN_LOOP() \
+	if (nih_main_loop_functions) { \
+		nih_free (nih_main_loop_functions); \
+		nih_main_loop_functions = NULL; \
+	} \
+	if (nih_child_watches) { \
+		nih_free (nih_child_watches); \
+		nih_child_watches = NULL; \
+	} \
+	if (nih_timers) { \
+		nih_free (nih_timers); \
+		nih_timers = NULL; \
+	} \
+	nih_child_init (); \
+	nih_main_loop_init (); \
+	nih_timer_init (); \
+	nih_io_init ()
+
+
+/**
  * obj_string_check:
  *
  * @a: first object,
@@ -702,10 +775,10 @@
 int string_check (const char *a, const char *b)
 	__attribute__ ((warn_unused_result));
 
-const char * get_upstart_binary (void)
+const char *get_upstart_binary (void)
 	__attribute__ ((warn_unused_result));
 
-const char * get_initctl_binary (void)
+const char *get_initctl_binary (void)
 	__attribute__ ((warn_unused_result));
 
 int strcmp_compar (const void *a, const void *b)
@@ -731,4 +804,17 @@
 
 void test_common_cleanup (void);
 
+void timer_cb (void *data, NihTimer *timer);
+
+void test_job_process_handler (void *data, pid_t pid,
+			  NihChildEvents event, int status);
+
+void test_main_loop_func (void *data, NihMainLoopFunc *self);
+
+int fd_valid (int fd)
+	__attribute__ ((warn_unused_result));
+
+NihIoBuffer *read_from_fd (void *parent, int fd)
+	__attribute__ ((warn_unused_result));
+
 #endif /* TEST_UTIL_COMMON_H */
 
=== modified file 'util/tests/test_initctl.c'
--- util/tests/test_initctl.c	2014-05-07 19:08:35 +0000
+++ util/tests/test_initctl.c	2014-05-22 13:31:23 +0000
@@ -11535,7 +11535,7 @@
 	TEST_NE_P (sessiondir, NULL);
 	
 	cmd = nih_sprintf (NULL, "rm %s/upstart/sessions/*.session 2>/dev/null", sessiondir);
-	system (cmd);
+	assert0 (system (cmd));
 
 	/* Use the "secret" interface */
 	TEST_EQ (setenv ("UPSTART_CONFDIR", confdir, 1), 0);