Merge ~cgrabowski/maas:service_boilerplate into ~cgrabowski/maas:rack_region_redesign_spike

[Christian Grabowski]

add supervisord services

add systemd service

add process services

add service supervisor

bump go version

[Alexsander de Souza (alexsander-souza)]

needs some fixes

review: Needs Fixing

[Christian Grabowski (cgrabowski)]

A few things I'm gonna change, and a few responses to things

[Alexsander de Souza (alexsander-souza)]

review: Approve

diff --git a/src/rackd_spike/go.mod b/src/rackd_spike/go.mod
index 2a2e641..99ccd82 100644
--- a/src/rackd_spike/go.mod
+++ b/src/rackd_spike/go.mod
@@ -1,8 +1,11 @@
 module rackd
 
-go 1.13
+go 1.16
 
 require (
+	github.com/coreos/go-systemd/v22 v22.3.2
+	github.com/godbus/dbus/v5 v5.0.4 // indirect
+	github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b // indirect
 	github.com/prometheus/client_golang v0.9.3
 	github.com/rs/zerolog v1.23.0
 	github.com/spf13/cobra v1.1.3
diff --git a/src/rackd_spike/go.sum b/src/rackd_spike/go.sum
index 65479d3..f7a0a9f 100644
--- a/src/rackd_spike/go.sum
+++ b/src/rackd_spike/go.sum
@@ -29,7 +29,9 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -45,6 +47,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -100,6 +103,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b h1:iNjcivnc6lhbvJA3LD622NPrUponluJrBWPIwGG/3Bg=
+github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
diff --git a/src/rackd_spike/internal/service/proc.go b/src/rackd_spike/internal/service/proc.go
new file mode 100644
index 0000000..ccc959b
--- /dev/null
+++ b/src/rackd_spike/internal/service/proc.go
@@ -0,0 +1,144 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"sync"
+	"syscall"
+)
+
+type ExecService struct {
+	sync.RWMutex
+	name   string
+	t      int
+	pid    int
+	cmd    *exec.Cmd
+	cmdStr string
+	args   []string
+}
+
+func NewExecService(name string, t int, cmd string, args ...string) Service {
+	return &ExecService{
+		name:   name,
+		t:      t,
+		pid:    -1,
+		cmdStr: cmd,
+		args:   args,
+	}
+}
+
+func (e *ExecService) Name() string {
+	return e.name
+}
+
+func (e *ExecService) Type() int {
+	return e.t
+}
+
+func (e *ExecService) PID() int {
+	e.RLock()
+	defer e.RUnlock()
+	return e.pid
+}
+
+func (e *ExecService) Start(ctx context.Context) (err error) {
+	e.Lock()
+	defer e.Unlock()
+	if e.pid != -1 && e.cmd != nil && !e.cmd.ProcessState.Exited() {
+		return ErrServiceAlreadyRunning
+	}
+	defer func() {
+		if err == nil {
+			e.pid = e.cmd.Process.Pid
+		}
+	}()
+	if e.cmd != nil && e.cmd.ProcessState.Exited() {
+		err = e.cmd.Process.Release()
+		if err != nil {
+			return err
+		}
+	}
+	e.cmd = exec.CommandContext(ctx, e.cmdStr, e.args...)
+	return e.cmd.Start()
+}
+
+func (e *ExecService) Stop(ctx context.Context) (err error) {
+	e.Lock()
+	defer e.Unlock()
+	if e.pid == -1 || e.cmd == nil || (e.cmd != nil && e.cmd.ProcessState != nil && e.cmd.ProcessState.Exited()) {
+		return ErrServiceAlreadyStopped
+	}
+	defer func() {
+		if err == nil {
+			e.pid = -1
+			e.cmd = nil
+		}
+	}()
+	procExitChan := make(chan error)
+	go func() {
+		select {
+		case <-ctx.Done():
+			close(procExitChan)
+			return
+		case procExitChan <- e.cmd.Wait():
+		}
+	}()
+	err = e.cmd.Process.Signal(syscall.SIGTERM)
+	if err != nil {
+		return err
+	}
+	select {
+	case <-ctx.Done():
+		err = e.cmd.Process.Kill()
+		if err != nil {
+			return err
+		}
+	case <-procExitChan:
+		// TODO handle if SIGTERM caused the process to shutdown uncleanly
+		return nil
+	}
+	return ctx.Err()
+}
+
+func (e *ExecService) Restart(ctx context.Context) (err error) {
+	err = e.Stop(ctx)
+	if err != nil {
+		return err
+	}
+	return e.Start(ctx)
+}
+
+func (e *ExecService) Status(_ context.Context) error {
+	e.RLock()
+	defer e.RUnlock()
+	if e.cmd.ProcessState != nil && (!e.cmd.ProcessState.Exited() || e.cmd.ProcessState.Success()) {
+		return nil
+	}
+	return fmt.Errorf("%w: service exited: %d", ErrUnexpectedServiceExit, e.cmd.ProcessState.ExitCode())
+}
+
+type ReloadableExecService struct {
+	ExecService
+	ReloadSig os.Signal
+}
+
+func NewReloadableExecService(sig os.Signal, name string, t int, cmd string, args ...string) ReloadableService {
+	return &ReloadableExecService{
+		ExecService: ExecService{
+			name:   name,
+			t:      t,
+			pid:    -1,
+			cmdStr: cmd,
+			args:   args,
+		},
+		ReloadSig: sig,
+	}
+}
+
+func (r *ReloadableExecService) Reload(_ context.Context) error {
+	r.RLock()
+	defer r.RUnlock()
+	return r.cmd.Process.Signal(r.ReloadSig)
+}
diff --git a/src/rackd_spike/internal/service/proc_test.go b/src/rackd_spike/internal/service/proc_test.go
new file mode 100644
index 0000000..520a3cf
--- /dev/null
+++ b/src/rackd_spike/internal/service/proc_test.go
@@ -0,0 +1,67 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+const testProc = "/usr/bin/yes"
+
+func TestExecService(t *testing.T) {
+	table := []struct {
+		Name string
+		In   struct {
+			Name string
+			Type int
+			Cmd  string
+			Args []string
+		}
+		Out        error
+		CleanupOut error
+		Action     string
+		Cleanup    string
+	}{{
+		Name: "start_new_service",
+		In: struct {
+			Name string
+			Type int
+			Cmd  string
+			Args []string
+		}{
+			Name: "mysvc",
+			Type: SvcDHCP,
+			Cmd:  testProc,
+			Args: []string{"hello"},
+		},
+		Action:  "start",
+		Cleanup: "stop",
+	}}
+	for _, tcase := range table {
+		t.Run(tcase.Name, func(tt *testing.T) {
+			svc := NewExecService(tcase.In.Name, tcase.In.Type, tcase.In.Cmd, tcase.In.Args...)
+			ctx := context.Background()
+			var err error
+			switch tcase.Action {
+			case "start":
+				err = svc.Start(ctx)
+			case "stop":
+				err = svc.Stop(ctx)
+			case "restart":
+				err = svc.Restart(ctx)
+			}
+			if !errors.Is(err, tcase.Out) {
+				tt.Fatalf("expected %v to equal %v", err, tcase.Out)
+			}
+			switch tcase.Cleanup {
+			case "stop":
+				err = svc.Stop(ctx)
+			default:
+				return
+			}
+			if !errors.Is(err, tcase.CleanupOut) {
+				tt.Fatalf("expectd %v to equal %v", err, tcase.CleanupOut)
+			}
+		})
+	}
+}
diff --git a/src/rackd_spike/internal/service/supervisor.go b/src/rackd_spike/internal/service/supervisor.go
new file mode 100644
index 0000000..8e5d308
--- /dev/null
+++ b/src/rackd_spike/internal/service/supervisor.go
@@ -0,0 +1,225 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"sync"
+)
+
+const (
+	SvcUnknown = iota
+	SvcDHCP
+	SvcDNS
+	SvcNTP
+	SvcPROXY
+)
+
+var (
+	ErrUnknownService        = errors.New("service is not registered with supervisor")
+	ErrServiceAlreadyRunning = errors.New("service is already in a running state")
+	ErrServiceAlreadyStopped = errors.New("service is already in a stopped state")
+	ErrInvalidServiceState   = errors.New("service is in an invalid state for this operation")
+	ErrUnexpectedServiceExit = errors.New("service exited unexpectedly")
+)
+
+type Service interface {
+	Name() string
+	Type() int
+	PID() int
+	Start(context.Context) error
+	Stop(context.Context) error
+	Restart(context.Context) error
+	Status(context.Context) error
+}
+
+type ReloadableService interface {
+	Service
+	Reload(context.Context) error
+}
+
+type SvcManager interface {
+	RegisterService(Service)
+	StartAll(context.Context) error
+	Start(context.Context, string) error
+	StartByType(context.Context, int) error
+	StopAll(context.Context) error
+	Stop(context.Context, string) error
+	StopByType(context.Context, int) error
+	Restart(context.Context, string) error
+	RestartByType(context.Context, int) error
+	Get(string) (Service, error)
+	GetType(int) ([]Service, error)
+	GetPID(int) (Service, error)
+}
+
+type Supervisor struct {
+	sync.RWMutex
+	procsByPID  map[int]Service
+	procsByName map[string]Service
+	procsByType map[int][]Service
+}
+
+func NewSupervisor() *Supervisor {
+	return &Supervisor{
+		procsByPID:  make(map[int]Service),
+		procsByName: make(map[string]Service),
+		procsByType: make(map[int][]Service),
+	}
+}
+
+func (s *Supervisor) RegisterService(svc Service) {
+	s.procsByName[svc.Name()] = svc
+	svcs := s.procsByType[svc.Type()]
+	s.procsByType[svc.Type()] = append(svcs, svc)
+	if pid := svc.PID(); pid != -1 {
+		s.procsByPID[pid] = svc
+	}
+}
+
+func (s *Supervisor) StartAll(ctx context.Context) error {
+	for _, svc := range s.procsByName {
+		err := svc.Start(ctx)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Supervisor) StopAll(ctx context.Context) error {
+	for _, svc := range s.procsByPID {
+		err := svc.Stop(ctx)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Supervisor) Start(ctx context.Context, name string) (err error) {
+	svc, err := s.Get(name)
+	if err != nil {
+		return err
+	}
+	pid := svc.PID()
+	if _, err = s.GetByPID(pid); err == nil || pid != -1 {
+		return ErrServiceAlreadyRunning
+	}
+	defer func() {
+		if err == nil {
+			s.Lock()
+			defer s.Unlock()
+			s.procsByPID[svc.PID()] = svc
+		}
+	}()
+	return svc.Start(ctx)
+}
+
+func (s *Supervisor) StartByType(ctx context.Context, t int) error {
+	svcs, err := s.GetByType(t)
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcs {
+		err = s.Start(ctx, svc.Name())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Supervisor) Stop(ctx context.Context, name string) (err error) {
+	svc, err := s.Get(name)
+	if err != nil {
+		return err
+	}
+	pid := svc.PID()
+	if _, err = s.GetByPID(svc.PID()); err != nil || pid == -1 {
+		return ErrServiceAlreadyStopped
+	}
+	defer func() {
+		if err == nil {
+			s.Lock()
+			defer s.Unlock()
+			delete(s.procsByPID, pid)
+		}
+	}()
+	return svc.Stop(ctx)
+}
+
+func (s *Supervisor) StopByType(ctx context.Context, t int) error {
+	svcs, err := s.GetByType(t)
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcs {
+		err = s.Stop(ctx, svc.Name())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Supervisor) Restart(ctx context.Context, name string) (err error) {
+	svc, err := s.Get(name)
+	if err != nil {
+		return err
+	}
+	pid := svc.PID()
+	if _, hasPID := s.procsByPID[pid]; !hasPID || pid == -1 {
+		return ErrInvalidServiceState
+	}
+	defer func() {
+		if err == nil {
+			s.Lock()
+			defer s.Unlock()
+			// Restart should change PIDs
+			delete(s.procsByPID, pid)
+			s.procsByPID[svc.PID()] = svc
+		}
+	}()
+	return svc.Restart(ctx)
+}
+
+func (s *Supervisor) RestartByType(ctx context.Context, t int) error {
+	svcs, err := s.GetByType(t)
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcs {
+		err = s.Restart(ctx, svc.Name())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Supervisor) Get(name string) (Service, error) {
+	s.RLock()
+	defer s.RUnlock()
+	if svc, ok := s.procsByName[name]; ok {
+		return svc, nil
+	}
+	return nil, ErrUnknownService
+}
+
+func (s *Supervisor) GetByType(t int) ([]Service, error) {
+	s.RLock()
+	defer s.RUnlock()
+	if svcs, ok := s.procsByType[t]; ok {
+		return svcs, nil
+	}
+	return nil, ErrUnknownService
+}
+
+func (s *Supervisor) GetByPID(pid int) (Service, error) {
+	s.RLock()
+	defer s.RUnlock()
+	if svc, ok := s.procsByPID[pid]; ok {
+		return svc, nil
+	}
+	return nil, ErrUnknownService
+}
diff --git a/src/rackd_spike/internal/service/supervisor_test.go b/src/rackd_spike/internal/service/supervisor_test.go
new file mode 100644
index 0000000..7e65701
--- /dev/null
+++ b/src/rackd_spike/internal/service/supervisor_test.go
@@ -0,0 +1,170 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+type MockService struct {
+	Service
+	pid        int
+	name       string
+	t          int
+	startErr   error
+	stopErr    error
+	restartErr error
+}
+
+func (m MockService) Name() string {
+	return m.name
+}
+
+func (m MockService) PID() int {
+	return m.pid
+}
+
+func (m MockService) Type() int {
+	return m.t
+}
+
+func (m MockService) Start(ctx context.Context) error {
+	return m.startErr
+}
+
+func (m MockService) Stop(ctx context.Context) error {
+	return m.stopErr
+}
+
+func (m MockService) Restart(ctx context.Context) error {
+	return m.restartErr
+}
+
+func TestSupvisor(t *testing.T) {
+	table := []struct {
+		Name   string
+		In     Service
+		Out    error
+		Action string
+	}{{
+		Name: "start_new_valid_service",
+		In: MockService{
+			pid:  -1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "start",
+	}, {
+		Name: "start_all_new_valid_service",
+		In: MockService{
+			pid:  -1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "start_all",
+	}, {
+		Name: "start_type_new_valid_service",
+		In: MockService{
+			pid:  -1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "start_type",
+	}, {
+		Name: "start_started_service",
+		In: MockService{
+			pid:  1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Out:    ErrServiceAlreadyRunning,
+		Action: "start",
+	}, {
+		Name: "stop_started_service",
+		In: MockService{
+			pid:  1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "stop",
+	}, {
+		Name: "stop_stopped_service",
+		In: MockService{
+			pid:  -1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Out:    ErrServiceAlreadyStopped,
+		Action: "stop",
+	}, {
+		Name: "stop_type_started_service",
+		In: MockService{
+			pid:  1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "stop_type",
+	}, {
+		Name: "stop_all_started_service",
+		In: MockService{
+			pid:  1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "stop_all",
+	}, {
+		Name: "restart_started_service",
+		In: MockService{
+			pid:  1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "restart",
+	}, {
+		Name: "restart_stopped_service",
+		In: MockService{
+			pid:  -1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Out:    ErrInvalidServiceState,
+		Action: "restart",
+	}, {
+		Name: "restart_by_type_started_service",
+		In: MockService{
+			pid:  1,
+			name: "mysvc",
+			t:    SvcDHCP,
+		},
+		Action: "restart_by_type",
+	}}
+	for _, tcase := range table {
+		t.Run(tcase.Name, func(tt *testing.T) {
+			sup := NewSupervisor()
+			sup.RegisterService(tcase.In)
+			ctx := context.Background()
+			var err error
+			switch tcase.Action {
+			case "start_all":
+				err = sup.StartAll(ctx)
+			case "start":
+				err = sup.Start(ctx, tcase.In.Name())
+			case "start_by_type":
+				err = sup.StartByType(ctx, tcase.In.Type())
+			case "stop_all":
+				err = sup.StopAll(ctx)
+			case "stop":
+				err = sup.Stop(ctx, tcase.In.Name())
+			case "stop_by_type":
+				err = sup.StopByType(ctx, tcase.In.Type())
+			case "restart":
+				err = sup.Restart(ctx, tcase.In.Name())
+			case "restart_by_type":
+				err = sup.RestartByType(ctx, tcase.In.Type())
+			}
+			if !errors.Is(err, tcase.Out) {
+				tt.Fatalf("expected %v to equal %v", err, tcase.Out)
+			}
+		})
+	}
+}
diff --git a/src/rackd_spike/internal/service/supervisord.go b/src/rackd_spike/internal/service/supervisord.go
new file mode 100644
index 0000000..71e0e31
--- /dev/null
+++ b/src/rackd_spike/internal/service/supervisord.go
@@ -0,0 +1,182 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/rpc"
+	"strconv"
+	"sync"
+
+	"github.com/kolo/xmlrpc"
+)
+
+var (
+	xmlrpcConnOnce  = &sync.Once{}
+	xmlrpcCloseOnce = &sync.Once{}
+)
+
+var (
+	supConn *xmlrpc.Client
+)
+
+var (
+	ErrSupervisordPIDNotFound     = errors.New("error supervisord service pid not found")
+	ErrSupervisordUnsupportedType = errors.New("received an unsupported type from supervisord")
+)
+
+func getSupervisordConn(endpoint string) (*xmlrpc.Client, error) {
+	var err error
+	xmlrpcConnOnce.Do(func() {
+		supConn, err = xmlrpc.NewClient(endpoint, nil)
+	})
+	return supConn, err
+}
+
+func CloseSupervisordConn() error {
+	var err error
+	xmlrpcCloseOnce.Do(func() {
+		err = supConn.Close()
+	})
+	return err
+}
+
+type SupervisordService struct {
+	sync.RWMutex
+	conn *xmlrpc.Client
+	name string
+	pid  int
+	t    int
+}
+
+func NewSupervisordService(endpoint, name string, t int) (Service, error) {
+	conn, err := getSupervisordConn(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &SupervisordService{
+		conn: conn,
+		name: name,
+		t:    t,
+	}, nil
+}
+
+func (s *SupervisordService) Name() string {
+	return s.name
+}
+
+func (s *SupervisordService) Type() int {
+	return s.t
+}
+
+func (s *SupervisordService) PID() int {
+	s.RLock()
+	defer s.RUnlock()
+	return s.pid
+}
+
+func (s *SupervisordService) readPIDFromResult(res map[string]interface{}) error {
+	if pidField, ok := res["pid"]; ok {
+		s.Lock()
+		defer s.Unlock()
+		var err error
+		switch pid := pidField.(type) {
+		case int:
+			s.pid = pid
+		case int32:
+			s.pid = int(pid)
+		case int64:
+			s.pid = int(pid)
+		case float32:
+			s.pid = int(pid)
+		case float64:
+			s.pid = int(pid)
+		case string:
+			s.pid, err = strconv.Atoi(pid)
+			if err != nil {
+				s.pid = -1
+				return err
+			}
+		default:
+			return fmt.Errorf("%w: pid:%v", ErrSupervisordUnsupportedType, pid)
+		}
+		return nil
+	}
+	return ErrSupervisordPIDNotFound
+}
+
+func (s *SupervisordService) Start(ctx context.Context) error {
+	resChan := make(chan *rpc.Call)
+	s.conn.Go("startProcess", []interface{}{s.name}, nil, resChan)
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case call := <-resChan:
+		if call.Error != nil {
+			return call.Error
+		}
+		res, err := s.getProcInfo(ctx)
+		if err != nil {
+			return err
+		}
+		err = s.readPIDFromResult(res)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *SupervisordService) Stop(ctx context.Context) error {
+	resChan := make(chan *rpc.Call)
+	s.conn.Go("stopProcess", []interface{}{s.name}, nil, resChan)
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case call := <-resChan:
+		if call.Error != nil {
+			return call.Error
+		}
+		s.Lock()
+		defer s.Unlock()
+		s.pid = -1
+	}
+	return nil
+}
+
+func (s *SupervisordService) Restart(ctx context.Context) error {
+	err := s.Stop(ctx)
+	if err != nil {
+		return err
+	}
+	return s.Start(ctx)
+}
+
+func (s *SupervisordService) getProcInfo(ctx context.Context) (map[string]interface{}, error) {
+	resChan := make(chan *rpc.Call)
+	s.conn.Go("getProcessInfo", []interface{}{s.name}, nil, resChan)
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case call := <-resChan:
+		if call.Error != nil {
+			return nil, call.Error
+		}
+		res, ok := call.Reply.(map[string]interface{})
+		if !ok {
+			return nil, ErrSupervisordUnsupportedType
+		}
+		return res, nil
+	}
+}
+
+func (s *SupervisordService) Status(ctx context.Context) error {
+	info, err := s.getProcInfo(ctx)
+	if err != nil {
+		return err
+	}
+	if info["stop"] != 0 {
+		return fmt.Errorf("%w: service %s exited", ErrUnexpectedServiceExit, s.name)
+	}
+	return nil
+}
diff --git a/src/rackd_spike/internal/service/systemd.go b/src/rackd_spike/internal/service/systemd.go
new file mode 100644
index 0000000..dbe0117
--- /dev/null
+++ b/src/rackd_spike/internal/service/systemd.go
@@ -0,0 +1,188 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"sync"
+
+	"github.com/coreos/go-systemd/v22/dbus"
+)
+
+const (
+	startFailMode = "fail"
+	// TODO add more supported start modes
+)
+
+const (
+	resultDone       = "done"
+	resultCanceled   = "canceled"
+	resultTimeout    = "timeout"
+	resultFailed     = "failed"
+	resultIsolate    = "isolate"
+	resultIgnoreDeps = "ignore-dependencies"
+	resultIgnoreReqs = "ignore-requirements"
+)
+
+var (
+	systemdConnOnce  = &sync.Once{}
+	systemdCloseOnce = &sync.Once{}
+)
+
+var (
+	ErrSystemdStart           = errors.New("error starting systemd unit")
+	ErrSystemdStop            = errors.New("error stopping systemd unit")
+	ErrSystemdPIDNotFound     = errors.New("error looking up unit's pid in systemd")
+	ErrSystemdInvalidPropType = errors.New("error systemd property not of supported type")
+	ErrSystemdServiceNotFound = errors.New("error systemd service not found")
+	ErrSystemdBadServiceState = errors.New("error systemd service is not active")
+)
+
+var (
+	dbusConn *dbus.Conn
+)
+
+func getDBusConn(ctx context.Context) (*dbus.Conn, error) {
+	var err error
+	systemdConnOnce.Do(func() {
+		if os.Getuid() == 0 {
+			dbusConn, err = dbus.NewSystemConnectionContext(ctx)
+			if err != nil {
+				return
+			}
+		} else {
+			dbusConn, err = dbus.NewUserConnectionContext(ctx)
+			if err != nil {
+				return
+			}
+		}
+	})
+	return dbusConn, err
+}
+
+func CloseSystemdConn() {
+	systemdCloseOnce.Do(func() {
+		dbusConn.Close()
+	})
+}
+
+type SystemdService struct {
+	sync.RWMutex
+	conn     *dbus.Conn
+	name     string
+	t        int
+	pid      int
+	UnitFile string
+}
+
+func NewSystemdService(ctx context.Context, name string, t int, unitFile string) (ReloadableService, error) {
+	conn, err := getDBusConn(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &SystemdService{
+		conn:     conn,
+		name:     name,
+		t:        t,
+		UnitFile: unitFile,
+	}, nil
+}
+
+func (s *SystemdService) Name() string {
+	return s.name
+}
+
+func (s *SystemdService) Type() int {
+	return s.t
+}
+
+func (s *SystemdService) PID() int {
+	s.RLock()
+	defer s.RUnlock()
+	return s.pid
+}
+
+func (s *SystemdService) readPIDFromProps(ctx context.Context) error {
+	s.Lock()
+	defer s.Unlock()
+	props, err := s.conn.GetAllPropertiesContext(ctx, s.UnitFile)
+	if err != nil {
+		return err
+	}
+	if p, ok := props["ExecMainPID"]; ok {
+		switch pid := p.(type) {
+		case int:
+			s.pid = pid
+		case int32:
+			s.pid = int(pid)
+		case int64:
+			s.pid = int(pid)
+		case float32:
+			s.pid = int(pid)
+		case float64:
+			s.pid = int(pid)
+		case string:
+			s.pid, err = strconv.Atoi(pid)
+			if err != nil {
+				s.pid = -1
+				return err
+			}
+		default:
+			return fmt.Errorf("%w: ExecMainPID", ErrSystemdInvalidPropType)
+		}
+		return nil
+	}
+	return ErrSystemdPIDNotFound
+}
+
+func (s *SystemdService) Start(ctx context.Context) error {
+	// "fail" mode will error if process is already running or queued to run
+	_, err := s.conn.StartUnitContext(ctx, s.UnitFile, startFailMode, nil)
+	if err != nil {
+		return err
+	}
+	return s.readPIDFromProps(ctx)
+}
+
+func (s *SystemdService) Stop(ctx context.Context) error {
+	_, err := s.conn.StopUnitContext(ctx, s.UnitFile, startFailMode, nil)
+	if err != nil {
+		return err
+	}
+	s.Lock()
+	defer s.Unlock()
+	s.pid = -1
+	return nil
+}
+
+func (s *SystemdService) Restart(ctx context.Context) error {
+	_, err := s.conn.RestartUnitContext(ctx, s.UnitFile, startFailMode, nil)
+	if err != nil {
+		return err
+	}
+	return s.readPIDFromProps(ctx)
+}
+
+func (s *SystemdService) Status(ctx context.Context) error {
+	status, err := s.conn.ListUnitsByNamesContext(ctx, []string{s.UnitFile})
+	if err != nil {
+		return err
+	}
+	if len(status) == 0 {
+		return fmt.Errorf("%w: %s", ErrSystemdServiceNotFound, s.UnitFile)
+	}
+	if status[0].ActiveState == "active" || status[0].ActiveState == "running" {
+		return nil
+	}
+	return ErrSystemdBadServiceState
+}
+
+func (s *SystemdService) Reload(ctx context.Context) error {
+	_, err := s.conn.RestartUnitContext(ctx, s.UnitFile, startFailMode, nil)
+	if err != nil {
+		return err
+	}
+	return s.readPIDFromProps(ctx)
+}