Merge ~cgrabowski/maas:rack_spike_http into maas:rack_region_exploration

[Christian Grabowski]

fix build

fix authentication

fix existing services

close connection on error

start supervisor earlier

add get proxy call

configure http services

add reverse proxy

make https urls work

quiet down logs

successfully proxy http

fix existing services

forward requests

add http proxy

[MAAS Lander (maas-lander)]

UNIT TESTS
-b rack_spike_http lp:~cgrabowski/maas/+git/maas into -b rack_region_exploration lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/10593/console
COMMIT: c557636756d200ec77051f36feaca06e097c877e

review: Needs Fixing

54e4f18... by Christian Grabowski on 2021-07-27

fix linter issues

[MAAS Lander (maas-lander)]

UNIT TESTS
-b rack_spike_http lp:~cgrabowski/maas/+git/maas into -b rack_region_exploration lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 61116646d017559136b515dc7348826891984a56

review: Approve

30c08a9... by Christian Grabowski on 2021-07-28

use events set in other call

[MAAS Lander (maas-lander)]

UNIT TESTS
-b rack_spike_http lp:~cgrabowski/maas/+git/maas into -b rack_region_exploration lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 3c09c61450b62d1ec3d3d57a3d069dd72d5add0b

review: Approve

581634f... by Christian Grabowski on 2021-07-28

initialize allowed_cidrs list first

[MAAS Lander (maas-lander)]

UNIT TESTS
-b rack_spike_http lp:~cgrabowski/maas/+git/maas into -b rack_region_exploration lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 581634f038b336abefa7f9f862a311a106a4a573

review: Approve

[Alexsander de Souza (alexsander-souza)]

I think there's a typo, just fix it before landing this

review: Approve

dd8d76c... by Christian Grabowski on 2021-07-28

fix typo

[MAAS Lander (maas-lander)]

UNIT TESTS
-b rack_spike_http lp:~cgrabowski/maas/+git/maas into -b rack_region_exploration lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: dd8d76c96c704e489dadb54592f89f75baf83c3c

review: Approve

diff --git a/src/maasserver/rpc/capnp/region.py b/src/maasserver/rpc/capnp/region.py
index ae8cb64..cbc5f05 100644
--- a/src/maasserver/rpc/capnp/region.py
+++ b/src/maasserver/rpc/capnp/region.py
@@ -105,6 +105,7 @@ class RegionController(region.RegionController.Server):
         self.shim = RegionServer()
         self.rack_controllers = rack_controllers
         self.server = server
+        self.events = set()
         super(RegionController, self).__init__()
 
     def ping(self):
@@ -183,7 +184,8 @@ class RegionController(region.RegionController.Server):
         return None
 
     def getTimeConfiguration_context(self, context):
-        self.event = capnp.PromiseFulfillerPair()
+        event = capnp.PromiseFulfillerPair()
+        self.events.add(event)
         systemId = context.params.systemId
 
         def get_result(cfg):
@@ -200,7 +202,8 @@ class RegionController(region.RegionController.Server):
                 lst[i] = s
 
             context.results.resp = resp
-            self.event.fulfill()
+            event.fulfill()
+            self.events.remove(event)
 
         self.shim.get_time_configuration(systemId).addCallback(get_result)
         return self.event.promise
@@ -216,8 +219,27 @@ class RegionController(region.RegionController.Server):
         self.shim.get_dns_configuration(systemId).addCallback(get_result)
         return prom
 
-    def getProxyConfiguration(self, systemId):
-        return None
+    def getProxyConfiguration_context(self, context):
+        system_id = context.params.systemId
+        event = capnp.PromiseFulfillerPair()
+        self.events.add(event)
+
+        def get_results(cfg):
+            resp = controller.ProxyConfiguration()
+            resp.enabled = cfg.get("enabled", True)
+            resp.port = cfg.get("port")
+            allowed_cidrs = cfg.get("allowed_cidrs", [])
+            lst = resp.init("allowed_cidrs", len(allowed_cidrs))
+            for i, cidr in enumerate(allowed_cidrs):
+                lst[i] = cidr
+            resp.allowedCidrs = lst
+            resp.preferV4Proxy = cfg.get("prefer_v4_proxy", False)
+            context.results.proxyConfig = resp
+            event.fulfill()
+            self.events.remove(event)
+
+        self.shim.get_proxy_configuration(system_id).addCallback(get_results)
+        return event.promise
 
     def getSyslogConfiguration(self, systemId):
         return None
diff --git a/src/rackd_spike/cmd/rackd.go b/src/rackd_spike/cmd/rackd.go
index 5091571..1dd491c 100644
--- a/src/rackd_spike/cmd/rackd.go
+++ b/src/rackd_spike/cmd/rackd.go
@@ -15,6 +15,7 @@ import (
 	"rackd/cmd/subcommands"
 	"rackd/internal/config"
 	"rackd/internal/dhcp"
+	"rackd/internal/http"
 	machinehelpers "rackd/internal/machine_helpers"
 	"rackd/internal/metrics"
 	"rackd/internal/ntp"
@@ -22,6 +23,7 @@ import (
 	"rackd/internal/transport"
 	"rackd/pkg/authenticate"
 	"rackd/pkg/controller"
+	httprpc "rackd/pkg/http"
 	ntprpc "rackd/pkg/ntp"
 	"rackd/pkg/region"
 	"rackd/pkg/register"
@@ -77,8 +79,22 @@ func registerProxyServices(ctx context.Context, sup service.SvcManager) error {
 	if err != nil {
 		return err
 	}
+	httpProxy, err := http.NewProxy(ctx, "0.0.0.0", 80)
+	if err != nil {
+		return err
+	}
+	machineResourcesPath, err := machinehelpers.GetResourcesBinPath()
+	if err != nil {
+		return err
+	}
+	reverseProxy, err := http.NewReverseProxy(ctx, machineResourcesPath, "0.0.0.0", 5248)
+	if err != nil {
+		return err
+	}
 	sup.RegisterService(dhcp.NewRelaySvc())
 	sup.RegisterService(ntpProxy)
+	sup.RegisterService(httpProxy)
+	sup.RegisterService(reverseProxy)
 	return nil
 }
 
@@ -190,6 +206,13 @@ func runRoot(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	rpcMgr.AddClient(ctx, ntpClient)
+
+	proxyClient, err := httprpc.New(sup)
+	if err != nil {
+		return err
+	}
+	rpcMgr.AddClient(ctx, proxyClient)
+
 	rackController, err := controller.NewRackController(ctx, true, initRegion, sup)
 	if err != nil {
 		return err
@@ -201,7 +224,7 @@ func runRoot(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		err = region.GetRemoteConfig(ctx, rpcMgr)
+		err = region.GetRemoteConfig(ctx, rpcMgr, sup)
 		if err != nil {
 			return err
 		}
diff --git a/src/rackd_spike/go-lint.sh b/src/rackd_spike/go-lint.sh
index 6506442..c0e86ae 100755
--- a/src/rackd_spike/go-lint.sh
+++ b/src/rackd_spike/go-lint.sh
@@ -5,7 +5,8 @@ go vet ./... 2>&1 | grep -v \
     -e 'internal/dhcp/service.go:46' \
     -e 'internal/dhcp/service.go:74' \
     -e 'internal/dhcp/service.go:102' \
-    -e 'internal/dhcp/service.go:130'
+    -e 'internal/dhcp/service.go:130' \
+    -e 'internal/http/reverse.go:57'
 
 # invert grep result
-[ "$?" -ne "0" ]
\ No newline at end of file
+[ "$?" -ne "0" ]
diff --git a/src/rackd_spike/internal/http/proxy.go b/src/rackd_spike/internal/http/proxy.go
new file mode 100644
index 0000000..69ff494
--- /dev/null
+++ b/src/rackd_spike/internal/http/proxy.go
@@ -0,0 +1,381 @@
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	stdlibLog "log"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/rs/zerolog"
+
+	"rackd/internal/service"
+)
+
+var (
+	ErrCircularRequest    = errors.New("error circular request")
+	ErrNotInternalAddress = errors.New("received external request on internal endpoint")
+	ErrClientNotAllowed   = errors.New("client not found in allowed CIDRs")
+)
+
+type ProxyService interface {
+	service.Service
+	Configure(ctx context.Context, enabled, preferV4 bool, port int16, allowedCidrs []*net.IPNet) error
+}
+
+type Proxy struct {
+	sync.Mutex
+	srvr         *http.Server
+	client       *http.Client
+	opts         *ProxyOptions
+	selfAddr     *net.TCPAddr
+	enabled      bool
+	allowedCidrs []*net.IPNet
+	getCtx       func() context.Context
+}
+
+func NewProxy(ctx context.Context, bindAddr string, port int) (*Proxy, error) {
+	return NewProxyWithTLS(ctx, bindAddr, port, nil)
+}
+
+func NewProxyWithTLS(ctx context.Context, bindAddr string, port int, tlsCfg *tls.Config) (*Proxy, error) {
+	addr := net.JoinHostPort(bindAddr, strconv.Itoa(port))
+	return NewProxyWithOptions(ctx, ProxyOptions{
+		FrontendAddr:   addr,
+		FrontendTLSCfg: tlsCfg,
+	})
+}
+
+type ProxyOptions struct {
+	MaxHeaderSize            int
+	HTTP2Enabled             bool
+	FrontendTLSCfg           *tls.Config
+	ConnectTimeout           time.Duration
+	ReadTimeout              time.Duration
+	HdrRdTimeout             time.Duration
+	WriteTimeout             time.Duration
+	IdleTimeout              time.Duration
+	UpstreamTLSTimeout       time.Duration
+	UpstreamMaxIdleConns     int
+	UpstreamMaxConnsPerHost  int
+	WriteBufSize             int
+	ReadBufSize              int
+	FrontendAddr             string
+	BackendAddr              string
+	IgnoreUpstreamHostVerify bool
+	NoKeepAlive              bool
+	NoCompression            bool
+}
+
+func NewProxyWithOptions(ctx context.Context, opts ProxyOptions) (*Proxy, error) {
+	logger := zerolog.Ctx(ctx)
+	errorLogger := stdlibLog.New(logger, "http-proxy", stdlibLog.LstdFlags|stdlibLog.LUTC) // wrap the existing logger in the stdlib log for http.Server
+	var clientTLSCfg *tls.Config
+	if opts.IgnoreUpstreamHostVerify {
+		clientTLSCfg = &tls.Config{
+			InsecureSkipVerify: opts.IgnoreUpstreamHostVerify,
+		}
+	}
+	if len(opts.BackendAddr) == 0 {
+		opts.BackendAddr = "0.0.0.0"
+	}
+	backendAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(opts.BackendAddr, "0"))
+	if err != nil {
+		return nil, err
+	}
+	frontendAddr, err := net.ResolveTCPAddr("tcp", opts.FrontendAddr)
+	if err != nil {
+		return nil, err
+	}
+	httpDialer := &net.Dialer{
+		Timeout:   opts.ConnectTimeout,
+		LocalAddr: backendAddr,
+		Cancel:    ctx.Done(),
+	}
+	if opts.NoKeepAlive {
+		httpDialer.KeepAlive = -1
+	}
+	httpsDialer := &tls.Dialer{
+		NetDialer: httpDialer,
+		Config:    clientTLSCfg,
+	}
+	if opts.HTTP2Enabled {
+		opts.FrontendTLSCfg.NextProtos = []string{"h2"}
+	}
+	p := &Proxy{
+		srvr: &http.Server{
+			Addr:              opts.FrontendAddr,
+			TLSConfig:         opts.FrontendTLSCfg,
+			ReadTimeout:       opts.ReadTimeout,
+			ReadHeaderTimeout: opts.HdrRdTimeout,
+			WriteTimeout:      opts.WriteTimeout,
+			IdleTimeout:       opts.IdleTimeout,
+			ErrorLog:          errorLogger,
+			BaseContext: func(_ net.Listener) context.Context {
+				return ctx
+			},
+		},
+		client: &http.Client{
+			Transport: &http.Transport{
+				Proxy:                  nil,
+				DialContext:            httpDialer.DialContext,
+				DialTLSContext:         httpsDialer.DialContext,
+				TLSClientConfig:        clientTLSCfg,
+				TLSHandshakeTimeout:    opts.UpstreamTLSTimeout,
+				DisableKeepAlives:      opts.NoKeepAlive,
+				DisableCompression:     opts.NoCompression,
+				MaxIdleConns:           opts.UpstreamMaxIdleConns,
+				MaxConnsPerHost:        opts.UpstreamMaxConnsPerHost,
+				IdleConnTimeout:        opts.IdleTimeout,
+				ResponseHeaderTimeout:  opts.HdrRdTimeout,
+				MaxResponseHeaderBytes: int64(opts.MaxHeaderSize),
+				WriteBufferSize:        opts.WriteBufSize,
+				ReadBufferSize:         opts.ReadBufSize,
+				ForceAttemptHTTP2:      opts.HTTP2Enabled,
+			},
+		},
+		opts:     &opts,
+		selfAddr: frontendAddr,
+	}
+	p.srvr.Handler = p
+	return p, nil
+}
+
+func (p *Proxy) Name() string {
+	return "http_proxy"
+}
+
+func (p *Proxy) Type() int {
+	return service.SvcPROXY
+}
+
+func (p *Proxy) PID() int {
+	return os.Getpid()
+}
+
+func (p *Proxy) Start(ctx context.Context) (err error) {
+	if !p.enabled {
+		return nil
+	}
+	var listener net.Listener
+	if p.opts.FrontendTLSCfg != nil {
+		listener, err = tls.Listen("tcp", p.opts.FrontendAddr, p.opts.FrontendTLSCfg)
+	} else {
+		listener, err = net.Listen("tcp", p.opts.FrontendAddr)
+	}
+	if err != nil {
+		return err
+	}
+	p.getCtx = func() context.Context {
+		return ctx
+	}
+	go p.srvr.Serve(listener)
+	return nil
+}
+
+func (p *Proxy) Stop(ctx context.Context) error {
+	return p.srvr.Shutdown(ctx)
+}
+
+func (p *Proxy) checkForCircularReq(r *http.Request) (err error) {
+	port := r.URL.Port()
+	if len(port) == 0 {
+		if r.URL.Scheme == "https" {
+			port = "443"
+		} else {
+			port = "80"
+		}
+	}
+	var ips []net.IP
+	ip := net.ParseIP(r.URL.Hostname())
+	if ip == nil {
+		ips, err = net.DefaultResolver.LookupIP(p.getCtx(), "ip", r.URL.Hostname())
+		if err != nil {
+			return err
+		}
+	} else {
+		ips = []net.IP{ip}
+	}
+	for _, iaddr := range ips {
+		remoteAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(iaddr.String(), port))
+		if err != nil {
+			return err
+		}
+		if remoteAddr.String() == p.selfAddr.String() {
+			return ErrCircularRequest
+		}
+	}
+	return nil
+}
+
+func (p *Proxy) setForwardingHeaders(h http.Header, req *http.Request) {
+	h.Set("X-Forwarded-For", req.RemoteAddr)
+	h.Set("X-Forwarded-Host", req.Host)
+	h.Set("Host", req.Host)
+	h.Set("Forwarded", fmt.Sprintf("by=%s", p.selfAddr.String()))
+	h.Add("Forwarded", fmt.Sprintf("for=%s", req.RemoteAddr))
+	h.Add("Forwarded", fmt.Sprintf("host=%s", req.Host))
+	h.Add("Forwarded", fmt.Sprintf("proto=%s", req.URL.Scheme))
+	h.Set("Origin", fmt.Sprintf("%s://%s", req.URL.Scheme, req.URL.Host))
+}
+
+func (p *Proxy) handleNetError(ctx context.Context, req *http.Request, w http.ResponseWriter, err error) {
+	if err == net.ErrClosed || err == syscall.ECONNREFUSED {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	if err == syscall.ECONNRESET || err == syscall.EHOSTUNREACH || err == syscall.EIO || err == syscall.EPIPE {
+		w.WriteHeader(http.StatusBadGateway)
+		return
+	}
+	if timeout, ok := err.(net.Error); (ok && timeout.Timeout()) || err == syscall.ETIMEDOUT {
+		w.WriteHeader(http.StatusGatewayTimeout)
+		return
+	}
+	w.WriteHeader(http.StatusInternalServerError)
+}
+
+func (p *Proxy) handleResponseError(ctx context.Context, req *http.Request, resp *http.Response, w http.ResponseWriter, err error) {
+	logger := zerolog.Ctx(ctx)
+	if resp == nil {
+		logger.Err(err).Msgf("no response received")
+		p.handleNetError(ctx, req, w, err)
+		return
+	}
+	logger.Err(err).Msgf("forward error to: %s, status %d", req.URL.RequestURI(), resp.StatusCode)
+	switch resp.StatusCode {
+	case 500, 400, 401, 402, 403, 404, 406, 409, 410, 411, 412, 413, 414, 415, 416, 417, 422, 423, 424, 425, 428, 431, 451, 501, 507, 510, 511:
+		w.WriteHeader(resp.StatusCode)
+		return
+	case 502, 503, 504, 505, 506:
+		w.WriteHeader(http.StatusBadGateway)
+		return
+	default:
+		p.handleNetError(ctx, req, w, err)
+	}
+}
+
+func (p *Proxy) PassResponseHeaders(w http.ResponseWriter, resp *http.Response) {
+	for k, v := range resp.Header {
+		w.Header().Set(k, v[0])
+		if len(v) > 1 {
+			for _, val := range v[1:] {
+				w.Header().Add(k, val)
+			}
+		}
+	}
+}
+
+func (p *Proxy) checkIsInternal(r *http.Request) error {
+	hostPortSlice := strings.Split(r.RemoteAddr, ":")
+	remoteHost, _ := hostPortSlice[0], hostPortSlice[1]
+	for _, internalHost := range []string{"localhost", "127.0.0.1", "::1"} {
+		if remoteHost == internalHost {
+			return nil
+		}
+	}
+	return ErrNotInternalAddress
+}
+
+func (p *Proxy) checkIPAllowed(remoteAddr string) error {
+	remoteHost, _, err := net.SplitHostPort(remoteAddr)
+	if err != nil {
+		return err
+	}
+	remoteIP := net.ParseIP(remoteHost)
+	_, localNet, err := net.ParseCIDR("127.0.0.0/8")
+	if err != nil {
+		return err
+	}
+	for _, allowed := range append(p.allowedCidrs, localNet) {
+		if !allowed.Contains(remoteIP) {
+			return ErrClientNotAllowed
+		}
+	}
+	return nil
+}
+
+func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	logger := zerolog.Ctx(p.getCtx())
+	defer r.Body.Close()
+	err := p.checkIPAllowed(r.RemoteAddr)
+	if err != nil {
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+	fwdURL := r.URL.String()
+	if len(r.URL.Scheme) == 0 {
+		if r.URL.Port() == "443" {
+			fwdURL = "https:" + fwdURL
+		} else {
+			fwdURL = "http:" + fwdURL
+		}
+	}
+	logger.Debug().Msgf("%s requesting %s %s", r.RemoteAddr, r.Method, fwdURL)
+	err = p.checkForCircularReq(r)
+	if err != nil {
+		w.WriteHeader(http.StatusLoopDetected)
+		return
+	}
+	fwdReq, err := http.NewRequestWithContext(p.getCtx(), r.Method, fwdURL, r.Body)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	fwdReq.Header = r.Header.Clone()
+	p.setForwardingHeaders(fwdReq.Header, r)
+	resp, err := p.client.Do(fwdReq)
+	if err != nil {
+		p.handleResponseError(p.getCtx(), r, resp, w, err)
+		return
+	}
+	defer logger.Debug().Msgf("%s responding to %s, %s", fwdURL, r.RemoteAddr, resp.Status)
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		p.handleResponseError(p.getCtx(), r, resp, w, nil)
+		return
+	}
+	p.PassResponseHeaders(w, resp)
+	w.WriteHeader(resp.StatusCode)
+	_, err = io.Copy(w, resp.Body)
+	if err != nil {
+		logger := zerolog.Ctx(p.getCtx())
+		logger.Err(err).Msgf("writing body to %s", r.RemoteAddr)
+	}
+}
+
+func (p *Proxy) Restart(ctx context.Context) error {
+	err := p.Stop(ctx)
+	if err != nil {
+		return err
+	}
+	return p.Start(ctx)
+}
+
+func (p *Proxy) Status(_ context.Context) error {
+	return nil
+}
+
+func (p *Proxy) Configure(ctx context.Context, enabled, preferV4 bool, port int16, allowedCidrs []*net.IPNet) (err error) {
+	p.Lock()
+	defer p.Unlock()
+	p.enabled = enabled
+	// TODO resolve DNS for IPv4 first if preferV4
+
+	p.opts.FrontendAddr = net.JoinHostPort(p.selfAddr.IP.String(), strconv.Itoa(int(port)))
+	p.selfAddr, err = net.ResolveTCPAddr("tcp", p.opts.FrontendAddr)
+	if err != nil {
+		return err
+	}
+	p.allowedCidrs = allowedCidrs
+	return p.Restart(ctx)
+}
diff --git a/src/rackd_spike/internal/http/reverse.go b/src/rackd_spike/internal/http/reverse.go
new file mode 100644
index 0000000..53b801b
--- /dev/null
+++ b/src/rackd_spike/internal/http/reverse.go
@@ -0,0 +1,234 @@
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+
+	"github.com/rs/zerolog"
+
+	machinehelpers "rackd/internal/machine_helpers"
+	"rackd/internal/service"
+)
+
+var (
+	ErrNoUpstreams = errors.New("no upstreams available to proxy")
+)
+
+type RevProxyService interface {
+	service.Service
+	Configure(context.Context, []string) error
+}
+
+type ReverseProxy struct {
+	sync.Mutex
+	Proxy
+	upstreamRegions []*net.TCPAddr
+	resourcePath    string
+}
+
+func NewReverseProxy(ctx context.Context, machineResourcePath, bindAddr string, port int) (*ReverseProxy, error) {
+	return NewReverseProxyWithTLS(ctx, machineResourcePath, bindAddr, port, nil)
+}
+
+func NewReverseProxyWithTLS(ctx context.Context, machineResourcePath, bindAddr string, port int, tlsCfg *tls.Config) (*ReverseProxy, error) {
+	addr := net.JoinHostPort(bindAddr, strconv.Itoa(port))
+	return NewReverseProxyWithOptions(ctx, machineResourcePath, ProxyOptions{
+		FrontendAddr:   addr,
+		FrontendTLSCfg: tlsCfg,
+	})
+}
+
+func NewReverseProxyWithOptions(ctx context.Context, machineResourcePath string, opts ProxyOptions) (*ReverseProxy, error) {
+	p, err := NewProxyWithOptions(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+	r := &ReverseProxy{
+		Proxy: *p,
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/MAAS/", r.ServeMAAS)
+	mux.Handle("/machine-resources/", http.FileServer(http.FS(os.DirFS(machineResourcePath))))
+	mux.HandleFunc("/images/", r.ServeImages)
+	mux.HandleFunc("/log", r.ServeLog)
+	mux.HandleFunc("/", r.ServeBoot)
+	r.Proxy.srvr.Handler = mux
+	return r, nil
+}
+
+func (r *ReverseProxy) Name() string {
+	return "http_reverse_proxy"
+}
+
+func (p *ReverseProxy) pickRegionHost() (*net.TCPAddr, error) {
+	p.Lock()
+	defer p.Unlock()
+	if len(p.upstreamRegions) == 0 {
+		return nil, ErrNoUpstreams
+	}
+	region := p.upstreamRegions[0]
+	if len(p.upstreamRegions) > 1 {
+		p.upstreamRegions = append(p.upstreamRegions[1:], region) // place the selected at end of list for round robin-ing
+	}
+	return region, nil
+}
+
+func (p *ReverseProxy) proxyPass(ctx context.Context, fwdURL string, w http.ResponseWriter, r *http.Request) {
+	logger := zerolog.Ctx(ctx)
+	req, err := http.NewRequestWithContext(ctx, r.Method, fwdURL, r.Body)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+	p.setForwardingHeaders(req.Header, r)
+	resp, err := p.client.Do(req)
+	if err != nil {
+		p.handleResponseError(ctx, r, resp, w, err)
+		return
+	}
+	if resp.StatusCode >= 400 {
+		p.handleResponseError(ctx, r, resp, w, nil)
+		return
+	}
+	p.PassResponseHeaders(w, resp)
+	w.WriteHeader(resp.StatusCode)
+	_, err = io.Copy(w, resp.Body)
+	if err != nil {
+		logger.Err(err).Msgf("writing body to %s", r.RemoteAddr)
+	}
+}
+
+func (p *ReverseProxy) ServeMAAS(w http.ResponseWriter, r *http.Request) {
+	ctx := p.getCtx()
+	defer r.Body.Close()
+	regionAddr, err := p.pickRegionHost()
+	if err != nil {
+		w.WriteHeader(http.StatusBadGateway)
+		return
+	}
+	fwdURL := "http://" + regionAddr.String() + r.URL.RequestURI()
+	p.proxyPass(ctx, fwdURL, w, r)
+}
+
+func (p *ReverseProxy) ServeImages(w http.ResponseWriter, r *http.Request) {
+	ctx := p.getCtx()
+	authReq, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://"+p.opts.FrontendAddr+"/log"), http.NoBody)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	resp, err := p.client.Do(authReq)
+	if err != nil {
+		p.handleResponseError(ctx, r, resp, w, err)
+		return
+	}
+	if resp.StatusCode >= 400 {
+		p.handleResponseError(ctx, r, resp, w, nil)
+		return
+	}
+	resource := filepath.Join(p.resourcePath, r.URL.Path)
+	f, err := os.Open(resource)
+	if err != nil {
+		if err == os.ErrNotExist {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		if err == os.ErrPermission {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	defer f.Close()
+	w.WriteHeader(http.StatusOK)
+	_, err = io.Copy(w, f)
+	if err != nil {
+		logger := zerolog.Ctx(ctx)
+		logger.Err(err).Msgf("writing body to %s", r.RemoteAddr)
+	}
+}
+
+func (p *ReverseProxy) ServeLog(w http.ResponseWriter, r *http.Request) {
+	ctx := p.getCtx()
+	defer r.Body.Close()
+	err := p.checkIsInternal(r)
+	if err != nil {
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+	region, err := p.pickRegionHost()
+	if err != nil {
+		w.WriteHeader(http.StatusBadGateway)
+		return
+	}
+	regionLogURL := "http://" + net.JoinHostPort(region.IP.String(), "5249") + r.URL.RequestURI()
+	req, err := http.NewRequestWithContext(ctx, r.Method, regionLogURL, r.Body)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	req.Header = r.Header.Clone()
+	req.Header.Set("X-Original-URI", r.URL.String())
+	req.Header.Set("X-Original-Remote-IP", r.RemoteAddr)
+	resp, err := p.client.Do(req)
+	if err != nil {
+		p.handleResponseError(ctx, r, resp, w, err)
+		return
+	}
+	if resp.StatusCode >= 400 {
+		p.handleResponseError(ctx, r, resp, w, nil)
+		return
+	}
+	p.PassResponseHeaders(w, resp)
+	w.WriteHeader(resp.StatusCode)
+	_, err = io.Copy(w, r.Body)
+	if err != nil {
+		logger := zerolog.Ctx(ctx)
+		logger.Err(err).Msgf("writing body to %s", r.RemoteAddr)
+	}
+}
+
+func (p *ReverseProxy) ServeBoot(w http.ResponseWriter, r *http.Request) {
+	ctx := p.getCtx()
+	region, err := p.pickRegionHost()
+	if err != nil {
+		w.WriteHeader(http.StatusBadGateway)
+		return
+	}
+	fwdURL := "http://" + net.JoinHostPort(region.IP.String(), "5249") + r.URL.RequestURI()
+	p.proxyPass(ctx, fwdURL, w, r)
+}
+
+func (p *ReverseProxy) Configure(ctx context.Context, regions []string) (err error) {
+	p.resourcePath, err = machinehelpers.GetResourcesBinPath()
+	if err != nil {
+		return err
+	}
+	p.upstreamRegions = make([]*net.TCPAddr, len(regions))
+	for i, region := range regions {
+		regionURL, err := url.Parse(region)
+		if err != nil { // if not url, try as host or IP
+			p.upstreamRegions[i], err = net.ResolveTCPAddr("tcp", net.JoinHostPort(region, "5240"))
+			if err != nil {
+				return err
+			}
+		}
+		p.upstreamRegions[i], err = net.ResolveTCPAddr("tcp", regionURL.Host)
+		if err != nil {
+			return err
+		}
+	}
+	return p.Restart(ctx)
+}
diff --git a/src/rackd_spike/internal/service/supervisor.go b/src/rackd_spike/internal/service/supervisor.go
index 09ba25e..ca9bd36 100644
--- a/src/rackd_spike/internal/service/supervisor.go
+++ b/src/rackd_spike/internal/service/supervisor.go
@@ -24,6 +24,8 @@ var (
 	ErrInvalidServiceState   = errors.New("service is in an invalid state for this operation")
 	ErrInvalidServiceType    = errors.New("invalid service type for given service")
 	ErrUnexpectedServiceExit = errors.New("service exited unexpectedly")
+	ErrUnsuccessfulStart     = errors.New("not all services started correctly")
+	ErrUnsuccessfulStop      = errors.New("not all services shutdown correctly")
 )
 
 // Service is an interface outlining behavior to manage external services
diff --git a/src/rackd_spike/internal/transport/rpc.go b/src/rackd_spike/internal/transport/rpc.go
index 4298c93..b71cdc8 100644
--- a/src/rackd_spike/internal/transport/rpc.go
+++ b/src/rackd_spike/internal/transport/rpc.go
@@ -198,3 +198,14 @@ func (r *RPCManager) GetHandler(handlerName string) (RPCHandler, error) {
 	}
 	return h, nil
 }
+
+func (r *RPCManager) ConnsToString() []string {
+	res := make([]string, len(r.conns))
+
+	var idx int
+	for k := range r.conns { // map key should be remote address of conn
+		res[idx] = k
+		idx++
+	}
+	return res
+}
diff --git a/src/rackd_spike/pkg/http/proxy.go b/src/rackd_spike/pkg/http/proxy.go
new file mode 100644
index 0000000..caeede0
--- /dev/null
+++ b/src/rackd_spike/pkg/http/proxy.go
@@ -0,0 +1,110 @@
+package http
+
+import (
+	"context"
+	"net"
+	internal "rackd/internal/http"
+	"rackd/internal/metrics"
+	"rackd/internal/service"
+	"rackd/internal/transport"
+	"rackd/pkg/rpc"
+	"sync"
+)
+
+type Client interface {
+	transport.RPCClient
+	GetProxyConfiguration(context.Context, string) error
+}
+
+type CapnpClient struct {
+	sync.Mutex
+	svc            internal.ProxyService
+	clients        map[string]*rpc.RegionController
+	lastGoodClient string
+	needReset      bool
+}
+
+func New(sup service.SvcManager) (Client, error) {
+	svcs, err := sup.GetByType(service.SvcPROXY)
+	if err != nil {
+		return nil, err
+	}
+	svc, ok := svcs[0].(internal.ProxyService)
+	if !ok {
+
+	}
+	return &CapnpClient{
+		svc:     svc,
+		clients: make(map[string]*rpc.RegionController),
+	}, nil
+}
+
+func (c *CapnpClient) Name() string {
+	return "http"
+}
+
+func (c *CapnpClient) RegisterMetrics(registry *metrics.Registry) error {
+	// TODO
+	return nil
+}
+
+func (c *CapnpClient) SetupClient(ctx context.Context, client *transport.ConnWrapper) {
+	c.Lock()
+	defer c.Unlock()
+	c.clients[client.Conn.RemoteAddr().String()] = client.Capnp()
+}
+
+func (c *CapnpClient) getClient(reset bool) (*rpc.RegionController, error) {
+	c.Lock()
+	defer c.Unlock()
+	if len(c.clients) == 0 {
+		return nil, transport.ErrRPCClientNotFound
+	}
+	if !reset && len(c.lastGoodClient) > 0 {
+		return c.clients[c.lastGoodClient], nil
+	}
+	for k, v := range c.clients {
+		c.lastGoodClient = k
+		return v, nil
+	}
+	return nil, transport.ErrRPCClientNotFound
+}
+
+func (c *CapnpClient) GetProxyConfiguration(ctx context.Context, systemID string) error {
+	client, err := c.getClient(c.needReset)
+	if err != nil {
+		return err
+	}
+	resp, release := client.GetProxyConfiguration(ctx, func(params rpc.RegionController_getProxyConfiguration_Params) error {
+		return params.SetSystemId(systemID)
+	})
+	defer release()
+	cfgResp, err := resp.Struct()
+	if err != nil {
+		return err
+	}
+	cfg, err := cfgResp.ProxyConfig()
+	if err != nil {
+		return err
+	}
+	enabled := cfg.Enabled()
+	preferV4 := cfg.PreferV4Proxy()
+	port := cfg.Port()
+	allowedCidrsProto, err := cfg.AllowedCidrs()
+	if err != nil {
+		return err
+	}
+	allowedCidrs := make([]*net.IPNet, allowedCidrsProto.Len())
+	for i := 0; i < allowedCidrsProto.Len(); i++ {
+		cidrProto, err := allowedCidrsProto.At(i)
+		if err != nil {
+			return err
+		}
+		_, cidr, err := net.ParseCIDR(cidrProto)
+		if err != nil {
+			return err
+		}
+		allowedCidrs[i] = cidr
+	}
+	return c.svc.Configure(ctx, enabled, preferV4, port, allowedCidrs)
+}
diff --git a/src/rackd_spike/pkg/region/config.go b/src/rackd_spike/pkg/region/config.go
index 2ad76a9..c199c91 100644
--- a/src/rackd_spike/pkg/region/config.go
+++ b/src/rackd_spike/pkg/region/config.go
@@ -3,19 +3,53 @@ package region
 import (
 	"context"
 
+	"github.com/rs/zerolog"
+
 	"rackd/internal/config"
+	httpInternal "rackd/internal/http"
+	"rackd/internal/service"
 	"rackd/internal/transport"
+	"rackd/pkg/http"
 	"rackd/pkg/ntp"
 )
 
-func GetRemoteConfig(ctx context.Context, rpcMgr *transport.RPCManager) error {
+func GetRemoteConfig(ctx context.Context, rpcMgr *transport.RPCManager, sup service.SvcManager) error {
+	log := zerolog.Ctx(ctx)
+
 	ntpIface, err := rpcMgr.GetClient("ntp")
 	if err != nil {
 		return err
 	}
-	if ntp, ok := ntpIface.(ntp.Client); ok {
-		return ntp.GetTimeConfiguration(ctx, config.Config.SystemID)
+
+	proxyIface, err := rpcMgr.GetClient("proxy")
+	if err != nil {
+		return err
+	}
+
+	revProxyIface, err := sup.Get("http_reverse_proxy")
+	if err != nil {
+		return err
+	}
+
+	if n, ok := ntpIface.(ntp.Client); ok {
+		log.Info().Msg("configuring NTP")
+		err = n.GetTimeConfiguration(ctx, config.Config.SystemID)
+		if err != nil {
+			return err
+		}
 	}
+	if proxy, ok := proxyIface.(http.Client); ok {
+		log.Info().Msg("configuring proxy")
+		err = proxy.GetProxyConfiguration(ctx, config.Config.SystemID)
+		if err != nil {
+			return err
+		}
+	}
+	if revProxy, ok := revProxyIface.(httpInternal.RevProxyService); ok {
+		log.Info().Msg("configuring HTTP reverse proxy")
+		revProxy.Configure(ctx, rpcMgr.ConnsToString())
+	}
+
 	return nil
 	// TODO add other services that fetch config from region controller
 }