Merge ~kajiya/+git/google-guest-agent:kajiya/upstream into ~ubuntu-core-dev/+git/google-guest-agent:upstream

[Chloé Smith]

New upstream version 20240213.00

[Utkarsh Gupta (utkarsh)]

$ dput ubuntu ../google-guest-agent_20240213.00-0ubuntu1_source.changes
Uploading google-guest-agent using ftp to ubuntu (host: upload.ubuntu.com; directory: /ubuntu)
running supported-distribution: check whether the target distribution is currently supported (using distro-info)
{'allowed': ['release', 'proposed', 'backports', 'security'], 'known': ['release', 'proposed', 'updates', 'backports', 'security']}
running required-fields: check whether a field is present and non-empty in the changes file
running checksum: verify checksums before uploading
running suite-mismatch: check the target distribution for common errors
running check-debs: makes sure the upload contains a binary package
running gpg: check GnuPG signatures before the upload
Uploading google-guest-agent_20240213.00-0ubuntu1.dsc
Uploading google-guest-agent_20240213.00.orig.tar.gz
Uploading google-guest-agent_20240213.00-0ubuntu1.debian.tar.xz
Uploading google-guest-agent_20240213.00-0ubuntu1_source.buildinfo
Uploading google-guest-agent_20240213.00-0ubuntu1_source.changes

review: Approve

diff --git a/.gitignore b/.gitignore
index 2ceee21..85eb02e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,15 @@
-# ignore all built binaries
+# Ignore all built binaries.
 **/gce_workload_cert_refresh
+**/gce_workload_cert_refresh.exe
 **/google_authorized_keys
+**/google_authorized_keys.exe
 **/google_guest_agent
+**/google_guest_agent.exe
 **/google_metadata_script_runner
+**/google_metadata_script_runner.exe
+
+# Don't ignore new content to directories.
+!**/gce_workload_cert_refresh/
+!**/google_authorized_keys/
+!**/google_guest_agent/
+!**/google_metadata_script_runner/
\ No newline at end of file
diff --git a/OWNERS b/OWNERS
index edde10e..59fca61 100644
--- a/OWNERS
+++ b/OWNERS
@@ -3,14 +3,14 @@
 
 approvers:
   - a-crate
+  - ajorg
   - bkatyl
   - chaitanyakulkarni28
   - dorileo
   - drewhli
   - elicriffield
+  - gaughen
   - jjerger
   - karnvadaliya
   - koln67
-  - quintonamore
-  - vorakl
   - zmarano
diff --git a/THIRD_PARTY_LICENSES/cloud.google.com/go/LICENSE b/THIRD_PARTY_LICENSES/cloud.google.com/go/iam/LICENSE
similarity index 100%
rename from THIRD_PARTY_LICENSES/cloud.google.com/go/LICENSE
rename to THIRD_PARTY_LICENSES/cloud.google.com/go/iam/LICENSE
diff --git a/THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/LICENSE b/THIRD_PARTY_LICENSES/cloud.google.com/go/internal/LICENSE
similarity index 100%
rename from THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/LICENSE
rename to THIRD_PARTY_LICENSES/cloud.google.com/go/internal/LICENSE
index 65fb971..d645695 100644
--- a/THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/LICENSE
+++ b/THIRD_PARTY_LICENSES/cloud.google.com/go/internal/LICENSE
@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2020 Google Inc.
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
diff --git a/THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/LICENSE b/THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/LICENSE
index 65fb971..f49a4e1 100644
--- a/THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/LICENSE
+++ b/THIRD_PARTY_LICENSES/github.com/GoogleCloudPlatform/guest-agent/LICENSE
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2020 Google Inc.
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.
\ No newline at end of file
diff --git a/THIRD_PARTY_LICENSES/github.com/Microsoft/go-winio/LICENSE b/THIRD_PARTY_LICENSES/github.com/Microsoft/go-winio/LICENSE
new file mode 100644
index 0000000..b8b569d
--- /dev/null
+++ b/THIRD_PARTY_LICENSES/github.com/Microsoft/go-winio/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
diff --git a/THIRD_PARTY_LICENSES/golang.org/x/xerrors/LICENSE b/THIRD_PARTY_LICENSES/golang.org/x/xerrors/LICENSE
new file mode 100644
index 0000000..e4a47e1
--- /dev/null
+++ b/THIRD_PARTY_LICENSES/golang.org/x/xerrors/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2019 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/THIRD_PARTY_LICENSES/google.golang.org/genproto/LICENSE b/THIRD_PARTY_LICENSES/google.golang.org/genproto/LICENSE
deleted file mode 100644
index d645695..0000000
--- a/THIRD_PARTY_LICENSES/google.golang.org/genproto/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/THIRD_PARTY_LICENSES/software.sslmate.com/src/go-pkcs12/LICENSE b/THIRD_PARTY_LICENSES/software.sslmate.com/src/go-pkcs12/LICENSE
index 6ac6b11..bcecd3d 100644
--- a/THIRD_PARTY_LICENSES/software.sslmate.com/src/go-pkcs12/LICENSE
+++ b/THIRD_PARTY_LICENSES/software.sslmate.com/src/go-pkcs12/LICENSE
@@ -25,4 +25,4 @@ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/gce_workload_cert_refresh/main.go b/gce_workload_cert_refresh/main.go
index adb83aa..c3c0b7d 100644
--- a/gce_workload_cert_refresh/main.go
+++ b/gce_workload_cert_refresh/main.go
@@ -16,11 +16,15 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"path"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/GoogleCloudPlatform/guest-agent/metadata"
@@ -28,15 +32,28 @@ import (
 )
 
 const (
-	contentDirPrefix  = "/run/secrets/workload-spiffe-contents"
+	// trustAnchorsKey endpoint contains a set of trusted certificates for peer X.509 certificate chain validation.
+	trustAnchorsKey = "instance/gce-workload-certificates/trust-anchors"
+	// workloadIdentitiesKey endpoint contains identities managed by the GCE control plane. This contains the X.509 certificate and the private key for the VM's trust domain.
+	workloadIdentitiesKey = "instance/gce-workload-certificates/workload-identities"
+	// configStatusKey contains status and any errors in the config values provided via the VM metadata.
+	configStatusKey = "instance/gce-workload-certificates/config-status"
+	// enableWorkloadCertsKey is set to true as custom metadata to enable automatic provisioning of credentials.
+	enableWorkloadCertsKey = "instance/attributes/enable-workload-certificate"
+	// contentDirPrefix is used as prefx to create certificate directories on refresh as contentDirPrefix-<time>.
+	contentDirPrefix = "/run/secrets/workload-spiffe-contents"
+	// tempSymlinkPrefix is used as prefix to create temporary symlinks on refresh as tempSymlinkPrefix-<time> to content directories.
 	tempSymlinkPrefix = "/run/secrets/workload-spiffe-symlink"
-	symlink           = "/run/secrets/workload-spiffe-credentials"
-	programName       = "gce_workload_certs_refresh"
+	// symlink points to the directory with current GCE workload certificates and is always expected to be present.
+	symlink = "/run/secrets/workload-spiffe-credentials"
 )
 
 var (
 	// mdsClient is the client used to query Metadata server.
-	mdsClient *metadata.Client
+	mdsClient   metadata.MDSClientInterface
+	programName = path.Base(os.Args[0])
+	// timeNow returns current time, defining as variable allows the time to be stubbed during testing.
+	timeNow = func() string { return time.Now().Format(time.RFC3339) }
 )
 
 func init() {
@@ -48,138 +65,86 @@ func logFormat(e logger.LogEntry) string {
 	return fmt.Sprintf("%s: %s", now, e.Message)
 }
 
+// isEnabled returns true only if enable-workload-certificate metadata attribute is present and set to true.
+func isEnabled(ctx context.Context) bool {
+	resp, err := getMetadata(ctx, enableWorkloadCertsKey)
+	if err != nil {
+		logger.Debugf("Failed to get %q from MDS with error: %v", enableWorkloadCertsKey, err)
+		return false
+	}
+
+	return bytes.EqualFold(resp, []byte("true"))
+}
+
 func getMetadata(ctx context.Context, key string) ([]byte, error) {
 	// GCE Workload Certificate endpoints return 412 Precondition failed if the VM was
 	// never configured with valid config values at least once. Without valid config
 	// values GCE cannot provision the workload certificates.
 	resp, err := mdsClient.GetKey(ctx, key, nil)
 	if err != nil {
-		return []byte{}, fmt.Errorf("failed to GET %q from MDS with error: %w", key, err)
+		return nil, fmt.Errorf("failed to GET %q from MDS with error: %w", key, err)
 	}
 	return []byte(resp), nil
 }
 
 /*
 metadata key instance/gce-workload-certificates/workload-identities
-
-	{
-	 "status": "OK",
-	 "workloadCredentials": {
-	  "PROJECT_ID.svc.id.goog": {
-	   "metadata": {
-	    "workload_creds_dir_path": "/var/run/secrets/workload-spiffe-credentials"
-	   },
-	   "certificatePem": "-----BEGIN CERTIFICATE-----datahere-----END CERTIFICATE-----",
-	   "privateKeyPem": "-----BEGIN PRIVATE KEY-----datahere-----END PRIVATE KEY-----"
-	  }
-	 }
-	}
-*/
-
-// WorkloadIdentities represents Workload Identities in metadata.
-type WorkloadIdentities struct {
-	Status              string
-	WorkloadCredentials map[string]WorkloadCredential
-}
-
-// UnmarshalJSON is a custom JSON unmarshaller for WorkloadIdentities.
-func (wi *WorkloadIdentities) UnmarshalJSON(b []byte) error {
-	tmp := map[string]json.RawMessage{}
-	err := json.Unmarshal(b, &tmp)
-	if err != nil {
-		return err
-	}
-
-	if err := json.Unmarshal(tmp["status"], &wi.Status); err != nil {
-		return err
-	}
-
-	wi.WorkloadCredentials = map[string]WorkloadCredential{}
-	wcs := map[string]json.RawMessage{}
-	if err := json.Unmarshal(tmp["workloadCredentials"], &wcs); err != nil {
-		return err
-	}
-
-	for domain, value := range wcs {
-		wc := WorkloadCredential{}
-		err := json.Unmarshal(value, &wc)
-		if err != nil {
-			return err
+MANAGED_WORKLOAD_IDENTITY_SPIFFE is of the format:
+spiffe://POOL_ID.global.PROJECT_NUMBER.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID
+
+{
+	"status": "OK", // Status of the response,
+	"workloadCredentials": { // Credentials for the VM's trust domains
+		"MANAGED_WORKLOAD_IDENTITY_SPIFFE": {
+			"certificatePem": "-----BEGIN CERTIFICATE-----datahere-----END CERTIFICATE-----",
+			"privateKeyPem": "-----BEGIN PRIVATE KEY-----datahere-----END PRIVATE KEY-----"
 		}
-		wi.WorkloadCredentials[domain] = wc
 	}
-
-	return nil
 }
+*/
 
 // WorkloadCredential represents Workload Credentials in metadata.
 type WorkloadCredential struct {
-	Metadata       Metadata
-	CertificatePem string
-	PrivateKeyPem  string
+	CertificatePem string `json:"certificatePem"`
+	PrivateKeyPem  string `json:"privateKeyPem"`
 }
 
-/*
-metadata key instance/gce-workload-certificates/root-certs
-
-	{
-	 "status": "OK",
-	 "rootCertificates": {
-	  "PROJECT.svc.id.goog": {
-	   "metadata": {
-	    "workload_creds_dir_path": "/var/run/secrets/workload-spiffe-credentials"
-	   },
-	   "rootCertificatesPem": "-----BEGIN CERTIFICATE-----datahere-----END CERTIFICATE-----"
-	  }
-	 }
-	}
-*/
-
-// WorkloadTrustedRootCerts represents Workload Trusted Root Certs in metadata.
-type WorkloadTrustedRootCerts struct {
-	Status           string
-	RootCertificates map[string]RootCertificate
+// WorkloadIdentities represents Workload Identities in metadata.
+type WorkloadIdentities struct {
+	Status              string                        `json:"status"`
+	WorkloadCredentials map[string]WorkloadCredential `json:"workloadCredentials"`
 }
 
-// UnmarshalJSON is a custom JSON unmarshaller for WorkloadTrustedRootCerts
-func (wtrc *WorkloadTrustedRootCerts) UnmarshalJSON(b []byte) error {
-	tmp := map[string]json.RawMessage{}
-	err := json.Unmarshal(b, &tmp)
-	if err != nil {
-		return err
-	}
-
-	if err := json.Unmarshal(tmp["status"], &wtrc.Status); err != nil {
-		return err
-	}
-
-	wtrc.RootCertificates = map[string]RootCertificate{}
-	rcs := map[string]json.RawMessage{}
-	if err := json.Unmarshal(tmp["rootCertificates"], &rcs); err != nil {
-		return err
-	}
-
-	for domain, value := range rcs {
-		rc := RootCertificate{}
-		err := json.Unmarshal(value, &rc)
-		if err != nil {
-			return err
-		}
-		wtrc.RootCertificates[domain] = rc
-	}
+/*
+metadata key instance/gce-workload-certificates/trust-anchors
+
+{
+    "status":  "<status string>" // Status of the response,
+    "trustAnchors": {  // Trust bundle for the VM's trust domains
+        "PEER_SPIFFE_TRUST_DOMAIN_1": {
+            "trustAnchorsPem" : "<Trust bundle containing the X.509 roots certificates>",
+		},
+        "PEER_SPIFFE_TRUST_DOMAIN_2": {
+            "trustAnchorsPem" : "<Trust bundle containing the X.509 roots certificates>",
+        }
+    }
+}
+*/
 
-	return nil
+// TrustAnchor represents one or more certificates in an arbitrary order in the metadata.
+type TrustAnchor struct {
+	TrustAnchorsPem string `json:"trustAnchorsPem"`
 }
 
-// RootCertificate represents a Root Certificate in metadata
-type RootCertificate struct {
-	Metadata            Metadata
-	RootCertificatesPem string
+// WorkloadTrustedAnchors represents Workload Trusted Root Certs in metadata.
+type WorkloadTrustedAnchors struct {
+	Status       string                 `json:"status"`
+	TrustAnchors map[string]TrustAnchor `json:"trustAnchors"`
 }
 
-// Metadata represents Metadata in metadata
-type Metadata struct {
-	WorkloadCredsDirPath string
+// outputOpts is a struct for output directory name and symlink templates.
+type outputOpts struct {
+	contentDirPrefix, tempSymlinkPrefix, symlink string
 }
 
 func main() {
@@ -193,37 +158,102 @@ func main() {
 	}
 
 	opts.Writers = []io.Writer{os.Stderr}
-	logger.Init(ctx, opts)
+
+	if err := logger.Init(ctx, opts); err != nil {
+		fmt.Printf("Error initializing logger: %v", err)
+		os.Exit(1)
+	}
+
 	defer logger.Infof("Done")
 
-	// TODO: prune old dirs
-	if err := refreshCreds(ctx); err != nil {
+	if !isEnabled(ctx) {
+		logger.Debugf("GCE Workload Certificate refresh is not enabled, skipping cert generation.")
+		return
+	}
+
+	out := outputOpts{contentDirPrefix, tempSymlinkPrefix, symlink}
+	if err := refreshCreds(ctx, out); err != nil {
 		logger.Fatalf("Error refreshCreds: %v", err.Error())
 	}
 
 }
 
-func refreshCreds(ctx context.Context) error {
-	project, err := getMetadata(ctx, "project/project-id")
+// findDomain finds the anchor matching with the domain from spiffeID.
+// spiffeID is of the form -
+// spiffe://POOL_ID.global.PROJECT_NUMBER.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID
+// where domain is POOL_ID.global.PROJECT_NUMBER.workload.id.goog
+// anchors is a map of various domains and their corresponding trust PEMs.
+// However, if anchor map contains single entry it returns that without any check.
+func findDomain(anchors map[string]TrustAnchor, spiffeID string) (string, error) {
+	c := len(anchors)
+	for k := range anchors {
+		if c == 1 {
+			return k, nil
+		}
+		if strings.Contains(spiffeID, k) {
+			return k, nil
+		}
+	}
+
+	return "", fmt.Errorf("no matching trust anchor found")
+}
+
+// writeTrustAnchors parses the input data, finds the domain from spiffeID and writes ca_certificate.pem
+// in the destDir for that domain.
+func writeTrustAnchors(wtrcsMd []byte, destDir, spiffeID string) error {
+	wtrcs := WorkloadTrustedAnchors{}
+	if err := json.Unmarshal(wtrcsMd, &wtrcs); err != nil {
+		return fmt.Errorf("error unmarshaling workload trusted root certs: %v", err)
+	}
+
+	// Currently there's only one trust anchor but there could be multipe trust anchors in future.
+	// In either case we want the trust anchor with domain matching with the one in SPIFFE ID.
+	domain, err := findDomain(wtrcs.TrustAnchors, spiffeID)
 	if err != nil {
-		return fmt.Errorf("error getting project ID: %v", err)
+		return err
 	}
 
+	return os.WriteFile(fmt.Sprintf("%s/ca_certificates.pem", destDir), []byte(wtrcs.TrustAnchors[domain].TrustAnchorsPem), 0644)
+}
+
+// writeWorkloadIdentities parses the input data, writes the certificates.pem, private_key.pem files in the
+// destDir, and returns the SPIFFE ID for which it wrote the certificates.
+func writeWorkloadIdentities(destDir string, wisMd []byte) (string, error) {
+	var spiffeID string
+	wis := WorkloadIdentities{}
+	if err := json.Unmarshal(wisMd, &wis); err != nil {
+		return "", fmt.Errorf("error unmarshaling workload identities response: %w", err)
+	}
+
+	// Its guaranteed to have single entry in workload credentials map.
+	for k := range wis.WorkloadCredentials {
+		spiffeID = k
+		break
+	}
+
+	if err := os.WriteFile(filepath.Join(destDir, "certificates.pem"), []byte(wis.WorkloadCredentials[spiffeID].CertificatePem), 0644); err != nil {
+		return "", fmt.Errorf("error writing certificates.pem: %w", err)
+	}
+
+	if err := os.WriteFile(filepath.Join(destDir, "private_key.pem"), []byte(wis.WorkloadCredentials[spiffeID].PrivateKeyPem), 0644); err != nil {
+		return "", fmt.Errorf("error writing private_key.pem: %w", err)
+	}
+	return spiffeID, nil
+}
+
+func refreshCreds(ctx context.Context, opts outputOpts) error {
+	now := timeNow()
+	contentDir := fmt.Sprintf("%s-%s", opts.contentDirPrefix, now)
+	tempSymlink := fmt.Sprintf("%s-%s", opts.tempSymlinkPrefix, now)
+
 	// Get status first so it can be written even when other endpoints are empty.
-	certConfigStatus, err := getMetadata(ctx, "instance/gce-workload-certificates/config-status")
+	certConfigStatus, err := getMetadata(ctx, configStatusKey)
 	if err != nil {
 		// Return success when certs are not configured to avoid unnecessary systemd failed units.
 		logger.Infof("Error getting config status, workload certificates may not be configured: %v", err)
 		return nil
 	}
 
-	domain := fmt.Sprintf("%s.svc.id.goog", project)
-	logger.Infof("Rotating workload credentials for trust domain %s", domain)
-
-	now := time.Now().Format(time.RFC3339)
-	contentDir := fmt.Sprintf("%s-%s", contentDirPrefix, now)
-	tempSymlink := fmt.Sprintf("%s-%s", tempSymlinkPrefix, now)
-
 	logger.Infof("Creating timestamp contents dir %s", contentDir)
 
 	if err := os.MkdirAll(contentDir, 0755); err != nil {
@@ -231,72 +261,59 @@ func refreshCreds(ctx context.Context) error {
 	}
 
 	// Write config_status first even if remaining endpoints are empty.
-	if err := os.WriteFile(fmt.Sprintf("%s/config_status", contentDir), certConfigStatus, 0644); err != nil {
+	if err := os.WriteFile(filepath.Join(contentDir, "config_status"), certConfigStatus, 0644); err != nil {
 		return fmt.Errorf("error writing config_status: %v", err)
 	}
 
 	// Handles the edge case where the config values provided for the first time may be invalid. This ensures
-	// that the symlink directory alwasys exists and contains the config_status to surface config errors to the VM.
-	if _, err := os.Stat(symlink); os.IsNotExist(err) {
+	// that the symlink directory always exists and contains the config_status to surface config errors to the VM.
+	if _, err := os.Stat(opts.symlink); os.IsNotExist(err) {
 		logger.Infof("Creating new symlink %s", symlink)
 
-		if err := os.Symlink(contentDir, symlink); err != nil {
+		if err := os.Symlink(contentDir, opts.symlink); err != nil {
 			return fmt.Errorf("error creating symlink: %v", err)
 		}
 	}
 
 	// Now get the rest of the content.
-	wisMd, err := getMetadata(ctx, "instance/gce-workload-certificates/workload-identities")
+	wisMd, err := getMetadata(ctx, workloadIdentitiesKey)
 	if err != nil {
 		return fmt.Errorf("error getting workload-identities: %v", err)
 	}
 
-	wtrcsMd, err := getMetadata(ctx, "instance/gce-workload-certificates/root-certs")
+	spiffeID, err := writeWorkloadIdentities(contentDir, wisMd)
 	if err != nil {
-		return fmt.Errorf("error getting workload-trusted-root-certs: %v", err)
+		return fmt.Errorf("failed to write workload identities with error: %w", err)
 	}
 
-	wis := WorkloadIdentities{}
-	if err := json.Unmarshal(wisMd, &wis); err != nil {
-		return fmt.Errorf("error unmarshaling workload identities response: %v", err)
-	}
-
-	wtrcs := WorkloadTrustedRootCerts{}
-	if err := json.Unmarshal(wtrcsMd, &wtrcs); err != nil {
-		return fmt.Errorf("error unmarshaling workload trusted root certs: %v", err)
-	}
-
-	if err := os.WriteFile(fmt.Sprintf("%s/certificates.pem", contentDir), []byte(wis.WorkloadCredentials[domain].CertificatePem), 0644); err != nil {
-		return fmt.Errorf("error writing certificates.pem: %v", err)
-	}
-
-	if err := os.WriteFile(fmt.Sprintf("%s/private_key.pem", contentDir), []byte(wis.WorkloadCredentials[domain].PrivateKeyPem), 0644); err != nil {
-		return fmt.Errorf("error writing private_key.pem: %v", err)
+	wtrcsMd, err := getMetadata(ctx, trustAnchorsKey)
+	if err != nil {
+		return fmt.Errorf("error getting workload-trust-anchors: %v", err)
 	}
 
-	if err := os.WriteFile(fmt.Sprintf("%s/ca_certificates.pem", contentDir), []byte(wtrcs.RootCertificates[domain].RootCertificatesPem), 0644); err != nil {
-		return fmt.Errorf("error writing ca_certificates.pem: %v", err)
+	if err := writeTrustAnchors(wtrcsMd, contentDir, spiffeID); err != nil {
+		return fmt.Errorf("failed to write trust anchors: %w", err)
 	}
 
 	if err := os.Symlink(contentDir, tempSymlink); err != nil {
 		return fmt.Errorf("error creating temporary link: %v", err)
 	}
 
-	oldTarget, err := os.Readlink(symlink)
+	oldTarget, err := os.Readlink(opts.symlink)
 	if err != nil {
 		logger.Infof("Error reading existing symlink: %v\n", err)
 		oldTarget = ""
 	}
 
 	// Only rotate on success of all steps above.
-	logger.Infof("Rotating symlink %s", symlink)
+	logger.Infof("Rotating symlink %s", opts.symlink)
 
-	if err := os.Rename(tempSymlink, symlink); err != nil {
+	if err := os.Rename(tempSymlink, opts.symlink); err != nil {
 		return fmt.Errorf("error rotating target link: %v", err)
 	}
 
 	// Clean up previous contents dir.
-	newTarget, err := os.Readlink(symlink)
+	newTarget, err := os.Readlink(opts.symlink)
 	if err != nil {
 		return fmt.Errorf("error reading new symlink: %v, unable to remove old symlink target", err)
 	}
diff --git a/gce_workload_cert_refresh/main_test.go b/gce_workload_cert_refresh/main_test.go
new file mode 100644
index 0000000..5edb1c8
--- /dev/null
+++ b/gce_workload_cert_refresh/main_test.go
@@ -0,0 +1,495 @@
+// Copyright 2023 Google LLC
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     https://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/guest-agent/metadata"
+	"github.com/google/go-cmp/cmp"
+)
+
+const (
+	workloadRespTpl = `
+	{
+		"status": "OK",
+		"workloadCredentials": {
+			"%s": {
+				"certificatePem": "%s",
+				"privateKeyPem": "%s"
+			}
+		}
+	}
+	`
+	trustAnchorRespTpl = `
+	{
+		"status": "Ok",
+		"trustAnchors": {
+			"%s": {
+				"trustAnchorsPem": "%s"
+			},
+			"%s": {
+				"trustAnchorsPem": "%s"
+			}
+		}
+	}
+	`
+	testConfigStatusResp = `
+	{
+		"status": "Ok",
+	}
+	`
+)
+
+func TestWorkloadIdentitiesUnmarshal(t *testing.T) {
+	certPem := "-----BEGIN CERTIFICATE-----datahere-----END CERTIFICATE-----"
+	pvtPem := "-----BEGIN PRIVATE KEY-----datahere-----END PRIVATE KEY-----"
+	spiffe := "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID"
+
+	resp := fmt.Sprintf(workloadRespTpl, spiffe, certPem, pvtPem)
+	want := WorkloadIdentities{
+		Status: "OK",
+		WorkloadCredentials: map[string]WorkloadCredential{
+			spiffe: {
+				CertificatePem: certPem,
+				PrivateKeyPem:  pvtPem,
+			},
+		},
+	}
+
+	got := WorkloadIdentities{}
+	if err := json.Unmarshal([]byte(resp), &got); err != nil {
+		t.Errorf("WorkloadIdentities.UnmarshalJSON(%s) failed unexpectedly with error: %v", resp, err)
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Workload identities diff (-want +got):\n%s", diff)
+	}
+}
+
+func TestTrustAnchorsUnmarshal(t *testing.T) {
+	domain1 := "12345.global.67890.workload.id.goog"
+	pem1 := "-----BEGIN CERTIFICATE-----datahere1-----END CERTIFICATE-----"
+	domain2 := "PEER_SPIFFE_TRUST_DOMAIN_2"
+	pem2 := "-----BEGIN CERTIFICATE-----datahere2-----END CERTIFICATE-----"
+
+	resp := fmt.Sprintf(trustAnchorRespTpl, domain1, pem1, domain2, pem2)
+	want := WorkloadTrustedAnchors{
+		Status: "Ok",
+		TrustAnchors: map[string]TrustAnchor{
+			domain1: {
+				TrustAnchorsPem: pem1,
+			},
+			domain2: {
+				TrustAnchorsPem: pem2,
+			},
+		},
+	}
+
+	got := WorkloadTrustedAnchors{}
+	if err := json.Unmarshal([]byte(resp), &got); err != nil {
+		t.Errorf("WorkloadTrustedRootCerts.UnmarshalJSON(%s) failed unexpectedly with error: %v", resp, err)
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Workload trusted anchors diff (-want +got):\n%s", diff)
+	}
+}
+
+func TestWriteTrustAnchors(t *testing.T) {
+	spiffe := "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID"
+	domain1 := "12345.global.67890.workload.id.goog"
+	pem1 := "-----BEGIN CERTIFICATE-----datahere1-----END CERTIFICATE-----"
+	domain2 := "PEER_SPIFFE_TRUST_DOMAIN_2"
+	pem2 := "-----BEGIN CERTIFICATE-----datahere2-----END CERTIFICATE-----"
+
+	resp := fmt.Sprintf(trustAnchorRespTpl, domain1, pem1, domain2, pem2)
+	dir := t.TempDir()
+	if err := writeTrustAnchors([]byte(resp), dir, spiffe); err != nil {
+		t.Errorf("writeTrustAnchors(%s,%s,%s) failed unexpectedly with error %v", resp, dir, spiffe, err)
+	}
+
+	got, err := os.ReadFile(filepath.Join(dir, "ca_certificates.pem"))
+	if err != nil {
+		t.Errorf("failed to read file at %s with error: %v", filepath.Join(dir, "ca_certificates.pem"), err)
+	}
+	if string(got) != pem1 {
+		t.Errorf("writeTrustAnchors(%s,%s,%s) wrote %q, expected to write %q", resp, dir, spiffe, string(got), pem1)
+	}
+}
+
+func TestWriteWorkloadIdentities(t *testing.T) {
+	certPem := "-----BEGIN CERTIFICATE-----datahere-----END CERTIFICATE-----"
+	pvtPem := "-----BEGIN PRIVATE KEY-----datahere-----END PRIVATE KEY-----"
+	spiffe := "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID"
+
+	resp := fmt.Sprintf(workloadRespTpl, spiffe, certPem, pvtPem)
+	dir := t.TempDir()
+
+	gotID, err := writeWorkloadIdentities(dir, []byte(resp))
+	if err != nil {
+		t.Errorf("writeWorkloadIdentities(%s,%s) failed unexpectedly with error %v", dir, resp, err)
+	}
+	if gotID != spiffe {
+		t.Errorf("writeWorkloadIdentities(%s,%s) = %s, want %s", dir, resp, gotID, spiffe)
+	}
+
+	gotCertPem, err := os.ReadFile(filepath.Join(dir, "certificates.pem"))
+	if err != nil {
+		t.Errorf("failed to read file at %s with error: %v", filepath.Join(dir, "certificates.pem"), err)
+	}
+	if string(gotCertPem) != certPem {
+		t.Errorf("writeWorkloadIdentities(%s,%s) wrote %q, expected to write %q", dir, resp, string(gotCertPem), certPem)
+	}
+
+	gotPvtPem, err := os.ReadFile(filepath.Join(dir, "private_key.pem"))
+	if err != nil {
+		t.Errorf("failed to read file at %s with error: %v", filepath.Join(dir, "private_key.pem"), err)
+	}
+	if string(gotPvtPem) != pvtPem {
+		t.Errorf("writeWorkloadIdentities(%s,%s) wrote %q, expected to write %q", dir, resp, string(gotPvtPem), pvtPem)
+	}
+}
+
+func TestFindDomainError(t *testing.T) {
+	anchors := map[string]TrustAnchor{
+		"67890.global.12345.workload.id.goog": {},
+		"55555.global.67890.workload.id.goog": {},
+	}
+	spiffeID := "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID"
+
+	if _, err := findDomain(anchors, spiffeID); err == nil {
+		t.Errorf("findDomain(%+v, %s) succeded for unknown anchors, want error", anchors, spiffeID)
+	}
+}
+
+func TestFindDomain(t *testing.T) {
+	tests := []struct {
+		desc     string
+		anchors  map[string]TrustAnchor
+		spiffeID string
+		want     string
+	}{
+		{
+			desc:     "single_trust_anchor",
+			anchors:  map[string]TrustAnchor{"12345.global.67890.workload.id.goog": {}},
+			spiffeID: "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID",
+			want:     "12345.global.67890.workload.id.goog",
+		},
+		{
+			desc: "multiple_trust_anchor",
+			anchors: map[string]TrustAnchor{
+				"67890.global.12345.workload.id.goog": {},
+				"12345.global.67890.workload.id.goog": {},
+			},
+			spiffeID: "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID",
+			want:     "12345.global.67890.workload.id.goog",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			got, err := findDomain(test.anchors, test.spiffeID)
+			if err != nil {
+				t.Errorf("findDomain(%+v, %s) failed unexpectedly with error: %v", test.anchors, test.spiffeID, err)
+			}
+			if got != test.want {
+				t.Errorf("findDomain(%+v, %s) = %s, want %s", test.anchors, test.spiffeID, got, test.want)
+			}
+		})
+	}
+}
+
+func TestIsEnabled(t *testing.T) {
+	ctx := context.Background()
+
+	tests := []struct {
+		desc    string
+		enabled string
+		want    bool
+		err     string
+	}{
+		{
+			desc:    "attr_correctly_added",
+			enabled: "true",
+			want:    true,
+		},
+		{
+			desc:    "attr_incorrectly_added",
+			enabled: "blaah",
+			want:    false,
+		},
+		{
+			desc: "attr_not_added",
+			want: false,
+			err:  enableWorkloadCertsKey,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			mdsClient = &mdsTestClient{enabled: test.enabled, throwErrOn: test.err}
+			if got := isEnabled(ctx); got != test.want {
+				t.Errorf("isEnabled(ctx) = %t, want %t", got, test.want)
+			}
+		})
+	}
+}
+
+// mdsTestClient is fake client to stub MDS response in unit tests.
+type mdsTestClient struct {
+	// Is credential generation enabled.
+	enabled string
+	// Workload template.
+	spiffe, certPem, pvtPem string
+	// Trust Anchor template.
+	domain1, pem1, domain2, pem2 string
+	// Throw error on MDS request for "key".
+	throwErrOn string
+}
+
+func (mds *mdsTestClient) Get(ctx context.Context) (*metadata.Descriptor, error) {
+	return nil, fmt.Errorf("Get() not yet implemented")
+}
+
+func (mds *mdsTestClient) GetKey(ctx context.Context, key string, headers map[string]string) (string, error) {
+	if mds.throwErrOn == key {
+		return "", fmt.Errorf("this is fake error for testing")
+	}
+
+	switch key {
+	case enableWorkloadCertsKey:
+		return mds.enabled, nil
+	case configStatusKey:
+		return testConfigStatusResp, nil
+	case workloadIdentitiesKey:
+		return fmt.Sprintf(workloadRespTpl, mds.spiffe, mds.certPem, mds.pvtPem), nil
+	case trustAnchorsKey:
+		return fmt.Sprintf(trustAnchorRespTpl, mds.domain1, mds.pem1, mds.domain2, mds.pem2), nil
+	default:
+		return "", fmt.Errorf("unknown key %q", key)
+	}
+}
+
+func (mds *mdsTestClient) GetKeyRecursive(ctx context.Context, key string) (string, error) {
+	return "", fmt.Errorf("GetKeyRecursive() not yet implemented")
+}
+
+func (mds *mdsTestClient) Watch(ctx context.Context) (*metadata.Descriptor, error) {
+	return nil, fmt.Errorf("Watch() not yet implemented")
+}
+
+func (mds *mdsTestClient) WriteGuestAttributes(ctx context.Context, key string, value string) error {
+	return fmt.Errorf("WriteGuestattributes() not yet implemented")
+}
+
+func TestRefreshCreds(t *testing.T) {
+	ctx := context.Background()
+	tmp := t.TempDir()
+
+	// Templates to use in iterations.
+	spiffeTpl := "spiffe://12345.global.67890.workload.id.goog.%d/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID"
+	domain1Tpl := "12345.global.67890.workload.id.goog.%d"
+	pem1Tpl := "-----BEGIN CERTIFICATE-----datahere1.%d-----END CERTIFICATE-----"
+	domain2 := "PEER_SPIFFE_TRUST_DOMAIN_2_IGNORE"
+	pem2Tpl := "-----BEGIN CERTIFICATE-----datahere2.%d-----END CERTIFICATE-----"
+	certPemTpl := "-----BEGIN CERTIFICATE-----datahere.%d-----END CERTIFICATE-----"
+	pvtPemTpl := "-----BEGIN PRIVATE KEY-----datahere.%d-----END PRIVATE KEY-----"
+
+	contentPrefix := filepath.Join(tmp, "workload-spiffe-contents")
+	tmpSymlinkPrefix := filepath.Join(tmp, "workload-spiffe-symlink")
+	link := filepath.Join(tmp, "workload-spiffe-credentials")
+	out := outputOpts{contentPrefix, tmpSymlinkPrefix, link}
+
+	// Run refresh creds thrice to test updates.
+	// Link (workload-spiffe-credentials) should always refer to the updated content
+	// and previous directories should be removed.
+	for i := 1; i <= 3; i++ {
+		timeNow = func() string { return fmt.Sprintf("%d", i) }
+		spiffe := fmt.Sprintf(spiffeTpl, i)
+		domain1 := fmt.Sprintf(domain1Tpl, i)
+		pem1 := fmt.Sprintf(pem1Tpl, i)
+		pem2 := fmt.Sprintf(pem2Tpl, i)
+		certPem := fmt.Sprintf(certPemTpl, i)
+		pvtPem := fmt.Sprintf(pvtPemTpl, i)
+
+		mdsClient = &mdsTestClient{
+			spiffe:  spiffe,
+			certPem: certPem,
+			pvtPem:  pvtPem,
+			domain1: domain1,
+			pem1:    pem1,
+			domain2: domain2,
+			pem2:    pem2,
+		}
+
+		if err := refreshCreds(ctx, out); err != nil {
+			t.Errorf("refreshCreds(ctx, %+v) failed unexpectedly with error: %v", out, err)
+		}
+
+		// Verify all files are created with the content as expected.
+		tests := []struct {
+			path    string
+			content string
+		}{
+			{
+				path:    filepath.Join(link, "ca_certificates.pem"),
+				content: pem1,
+			},
+			{
+				path:    filepath.Join(link, "certificates.pem"),
+				content: certPem,
+			},
+			{
+				path:    filepath.Join(link, "private_key.pem"),
+				content: pvtPem,
+			},
+			{
+				path:    filepath.Join(link, "config_status"),
+				content: testConfigStatusResp,
+			},
+		}
+
+		for _, test := range tests {
+			t.Run(test.path, func(t *testing.T) {
+				got, err := os.ReadFile(test.path)
+				if err != nil {
+					t.Errorf("failed to read expected file %q and content %q with error: %v", test.path, test.content, err)
+				}
+				if string(got) != test.content {
+					t.Errorf("refreshCreds(ctx, %+v) wrote %q, want content %q", out, string(got), test.content)
+				}
+			})
+		}
+
+		// Verify the symlink was created and references the right destination directory.
+		want := fmt.Sprintf("%s-%d", contentPrefix, i)
+		got, err := os.Readlink(link)
+		if err != nil {
+			t.Errorf("os.Readlink(%s) failed unexpectedly with error %v", link, err)
+		}
+		if got != want {
+			t.Errorf("os.Readlink(%s) = %s, want %s", link, got, want)
+		}
+
+		// If its not first run make sure prev creds are deleted.
+		if i > 1 {
+			prevDir := fmt.Sprintf("%s-%d", contentPrefix, i-1)
+			if _, err := os.Stat(prevDir); err == nil {
+				t.Errorf("os.Stat(%s) succeeded on prev content directory, want error", prevDir)
+			}
+		}
+	}
+}
+
+func TestRefreshCredsError(t *testing.T) {
+	ctx := context.Background()
+	tmp := t.TempDir()
+
+	// Templates to use in iterations.
+	spiffe := "spiffe://12345.global.67890.workload.id.goog/ns/NAMESPACE_ID/sa/MANAGED_IDENTITY_ID"
+	domain1 := "12345.global.67890.workload.id.goog"
+	pem1 := "-----BEGIN CERTIFICATE-----datahere1-----END CERTIFICATE-----"
+	domain2 := "PEER_SPIFFE_TRUST_DOMAIN_2_IGNORE"
+	pem2 := "-----BEGIN CERTIFICATE-----datahere2-----END CERTIFICATE-----"
+	certPem := "-----BEGIN CERTIFICATE-----datahere-----END CERTIFICATE-----"
+	pvtPem := "-----BEGIN PRIVATE KEY-----datahere-----END PRIVATE KEY-----"
+
+	contentPrefix := filepath.Join(tmp, "workload-spiffe-contents")
+	tmpSymlinkPrefix := filepath.Join(tmp, "workload-spiffe-symlink")
+	link := filepath.Join(tmp, "workload-spiffe-credentials")
+	out := outputOpts{contentPrefix, tmpSymlinkPrefix, link}
+
+	client := &mdsTestClient{
+		spiffe:  spiffe,
+		certPem: certPem,
+		pvtPem:  pvtPem,
+		domain1: domain1,
+		pem1:    pem1,
+		domain2: domain2,
+		pem2:    pem2,
+	}
+
+	mdsClient = client
+
+	// Run refresh creds twice. First run would succeed and second would fail. Verify all
+	// creds generated on the first run are present as is after failed second run.
+	for i := 1; i <= 2; i++ {
+		timeNow = func() string { return fmt.Sprintf("%d", i) }
+
+		if i == 1 {
+			// First run should succeed.
+			if err := refreshCreds(ctx, out); err != nil {
+				t.Errorf("refreshCreds(ctx, %+v) failed unexpectedly with error: %v", out, err)
+			}
+		} else if i == 2 {
+			// Second run should fail. Fail in getting last metadata entry.
+			client.throwErrOn = trustAnchorsKey
+			if err := refreshCreds(ctx, out); err == nil {
+				t.Errorf("refreshCreds(ctx, %+v) succeeded for fake metadata error, should've failed", out)
+			}
+		}
+
+		// Verify all files are created and are still present with the content as expected.
+		tests := []struct {
+			path    string
+			content string
+		}{
+			{
+				path:    filepath.Join(link, "ca_certificates.pem"),
+				content: pem1,
+			},
+			{
+				path:    filepath.Join(link, "certificates.pem"),
+				content: certPem,
+			},
+			{
+				path:    filepath.Join(link, "private_key.pem"),
+				content: pvtPem,
+			},
+			{
+				path:    filepath.Join(link, "config_status"),
+				content: testConfigStatusResp,
+			},
+		}
+
+		for _, test := range tests {
+			t.Run(test.path, func(t *testing.T) {
+				got, err := os.ReadFile(test.path)
+				if err != nil {
+					t.Errorf("failed to read expected file %q and content %q with error: %v", test.path, test.content, err)
+				}
+				if string(got) != test.content {
+					t.Errorf("refreshCreds(ctx, %+v) wrote %q, want content %q", out, string(got), test.content)
+				}
+			})
+		}
+
+		// Verify the symlink was created and references the same destination directory.
+		want := fmt.Sprintf("%s-%d", contentPrefix, 1)
+		got, err := os.Readlink(link)
+		if err != nil {
+			t.Errorf("os.Readlink(%s) failed unexpectedly with error %v", link, err)
+		}
+		if got != want {
+			t.Errorf("os.Readlink(%s) = %s, want %s", link, got, want)
+		}
+	}
+}
diff --git a/go.mod b/go.mod
index d5f8c28..1e4ea08 100644
--- a/go.mod
+++ b/go.mod
@@ -10,12 +10,14 @@ require (
 	github.com/go-ini/ini v1.66.6
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/golang/protobuf v1.5.3
+	github.com/google/go-cmp v0.5.9
 	github.com/google/go-tpm v0.9.0
 	github.com/google/go-tpm-tools v0.4.0
 	github.com/google/tink/go v1.7.0
 	github.com/kardianos/service v1.2.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07
+	golang.org/x/crypto v0.11.0
 	golang.org/x/sys v0.11.0
 	google.golang.org/grpc v1.57.0
 	google.golang.org/protobuf v1.31.0
@@ -29,7 +31,7 @@ require (
 	cloud.google.com/go/iam v1.1.1 // indirect
 	cloud.google.com/go/logging v1.7.0 // indirect
 	cloud.google.com/go/longrunning v0.5.1 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/google/go-sev-guest v0.7.0 // indirect
 	github.com/google/logger v1.1.1 // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
@@ -39,11 +41,12 @@ require (
 	github.com/pborman/uuid v1.2.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/net v0.12.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/api v0.134.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
diff --git a/go.sum b/go.sum
index 55c71c0..9c5c73b 100644
--- a/go.sum
+++ b/go.sum
@@ -17,6 +17,8 @@ cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/GoogleCloudPlatform/guest-logging-go v0.0.0-20230710215706-450679fd88a9 h1:b3geIwOPAShYtR4F0XFt+2NJXTHVTfbxUFmrpiZXHdQ=
 github.com/GoogleCloudPlatform/guest-logging-go v0.0.0-20230710215706-450679fd88a9/go.mod h1:6ZqSUIZRAPR5dNMWJ+FwIarFFQ9t5qalaKQs20o6h+I=
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -94,8 +96,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/kardianos/service v1.2.1 h1:AYndMsehS+ywIS6RB9KOlcXzteWUzxgMgBymJD7+BYk=
 github.com/kardianos/service v1.2.1/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
 github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -134,6 +138,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -176,6 +182,7 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -191,6 +198,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/google_authorized_keys/main.go b/google_authorized_keys/main.go
index 4961397..634ab5a 100644
--- a/google_authorized_keys/main.go
+++ b/google_authorized_keys/main.go
@@ -20,24 +20,27 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"net/http"
 	"os"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/GoogleCloudPlatform/guest-agent/metadata"
 	"github.com/GoogleCloudPlatform/guest-agent/utils"
 	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
 )
 
 var (
-	programName       = "GoogleAuthorizedKeysCommand"
-	metadataURL       = "http://169.254.169.254/computeMetadata/v1/"
-	metadataRecursive = "/?recursive=true&alt=json"
-	defaultTimeout    = 2 * time.Second
+	client      metadata.MDSClientInterface
+	programName = path.Base(os.Args[0])
 )
 
+func init() {
+	client = metadata.New()
+}
+
 func logFormat(e logger.LogEntry) string {
 	now := time.Now().Format("2006/01/02 15:04:05")
 	return fmt.Sprintf("%s %s: %s", now, programName, e.Message)
@@ -49,45 +52,6 @@ func logFormatWindows(e logger.LogEntry) string {
 	return fmt.Sprintf("%s %s: %s", now, programName, e.Message)
 }
 
-func getMetadata(key string, recurse bool) ([]byte, error) {
-	client := &http.Client{
-		Timeout: defaultTimeout,
-	}
-
-	url := metadataURL + key
-	if recurse {
-		url += metadataRecursive
-	}
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Add("Metadata-Flavor", "Google")
-
-	var res *http.Response
-	// Retry forever, increase sleep between retries (up to 5 times) in order
-	// to wait for slow network initialization.
-	var rt time.Duration
-	for i := 1; ; i++ {
-		res, err = client.Do(req)
-		if err == nil {
-			break
-		}
-		if i < 6 {
-			rt = time.Duration(3*i) * time.Second
-		}
-		logger.Errorf("error connecting to metadata server, retrying in %s, error: %v", rt, err)
-		time.Sleep(rt)
-	}
-	defer res.Body.Close()
-
-	md, err := io.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	return md, nil
-}
-
 func parseSSHKeys(username string, keys []string) []string {
 	var keyList []string
 	for _, key := range keys {
@@ -143,7 +107,7 @@ type attributes struct {
 	SSHKeys             []string
 }
 
-func getMetadataAttributes(metadataKey string) (*attributes, error) {
+func getMetadataAttributes(ctx context.Context, metadataKey string) (*attributes, error) {
 	var a attributes
 	type jsonAttributes struct {
 		EnableWindowsSSH    string `json:"enable-windows-ssh"`
@@ -151,11 +115,12 @@ func getMetadataAttributes(metadataKey string) (*attributes, error) {
 		SSHKeys             string `json:"ssh-keys"`
 	}
 	var ja jsonAttributes
-	metadata, err := getMetadata(metadataKey, true)
+	metadata, err := client.GetKeyRecursive(ctx, metadataKey)
 	if err != nil {
 		return nil, err
 	}
-	if err := json.Unmarshal(metadata, &ja); err != nil {
+
+	if err := json.Unmarshal([]byte(metadata), &ja); err != nil {
 		return nil, err
 	}
 
@@ -191,12 +156,12 @@ func main() {
 	}
 	logger.Init(ctx, opts)
 
-	instanceAttributes, err := getMetadataAttributes("instance/attributes/")
+	instanceAttributes, err := getMetadataAttributes(ctx, "instance/attributes/")
 	if err != nil {
 		logger.Errorf("Cannot read instance metadata attributes: %v", err)
 		os.Exit(1)
 	}
-	projectAttributes, err := getMetadataAttributes("project/attributes/")
+	projectAttributes, err := getMetadataAttributes(ctx, "project/attributes/")
 	if err != nil {
 		logger.Errorf("Cannot read project metadata attributes: %v", err)
 		os.Exit(1)
diff --git a/google_authorized_keys/main_test.go b/google_authorized_keys/main_test.go
index 49315f1..ea739a1 100644
--- a/google_authorized_keys/main_test.go
+++ b/google_authorized_keys/main_test.go
@@ -15,14 +15,15 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"net/http"
-	"net/http/httptest"
 	"reflect"
 	"strconv"
 	"strings"
 	"testing"
-	"time"
+
+	"github.com/GoogleCloudPlatform/guest-agent/metadata"
+	"github.com/GoogleCloudPlatform/guest-agent/utils"
 )
 
 func stringSliceEqual(a, b []string) bool {
@@ -50,16 +51,20 @@ var truebool *bool = &t
 var falsebool *bool = &f
 
 func TestParseSSHKeys(t *testing.T) {
+	pubKeyA := utils.MakeRandRSAPubKey(t)
+	pubKeyB := utils.MakeRandRSAPubKey(t)
+	pubKey := utils.MakeRandRSAPubKey(t)
+
 	keys := []string{
 		"# Here is some random data in the file.",
-		"usera:ssh-rsa AAAA1234USERA",
-		"userb:ssh-rsa AAAA1234USERB",
-		`usera:ssh-rsa AAAA1234 google-ssh {"userName":"[e-mail]","expireOn":"2095-04-23T12:34:56+0000"}`,
-		`usera:ssh-rsa AAAA1234 google-ssh {"userName":"[e-mail]","expireOn":"2020-04-23T12:34:56+0000"}`,
+		fmt.Sprintf("usera:ssh-rsa %s", pubKeyA),
+		fmt.Sprintf("userb:ssh-rsa %s", pubKeyB),
+		fmt.Sprintf(`usera:ssh-rsa %s google-ssh {"userName":"[e-mail]","expireOn":"2095-04-23T12:34:56+0000"}`, pubKey),
+		fmt.Sprintf(`usera:ssh-rsa %s google-ssh {"userName":"[e-mail]","expireOn":"2020-04-23T12:34:56+0000"}`, pubKey),
 	}
 	expected := []string{
-		"ssh-rsa AAAA1234USERA",
-		`ssh-rsa AAAA1234 google-ssh {"userName":"[e-mail]","expireOn":"2095-04-23T12:34:56+0000"}`,
+		fmt.Sprintf("ssh-rsa %s", pubKeyA),
+		fmt.Sprintf(`ssh-rsa %s google-ssh {"userName":"[e-mail]","expireOn":"2095-04-23T12:34:56+0000"}`, pubKey),
 	}
 
 	user := "usera"
@@ -117,6 +122,8 @@ func TestCheckWinSSHEnabled(t *testing.T) {
 }
 
 func TestGetUserKeysNew(t *testing.T) {
+	pubKey := utils.MakeRandRSAPubKey(t)
+
 	tests := []struct {
 		userName         string
 		instanceMetadata attributes
@@ -125,102 +132,114 @@ func TestGetUserKeysNew(t *testing.T) {
 	}{
 		{
 			userName: "name",
-			instanceMetadata: attributes{BlockProjectSSHKeys: false,
-				SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"},
+			instanceMetadata: attributes{
+				BlockProjectSSHKeys: false,
+				SSHKeys: []string{
+					fmt.Sprintf("name:ssh-rsa %s instance1", pubKey),
+					fmt.Sprintf("othername:ssh-rsa %s instance2", pubKey),
+				},
 			},
 			projectMetadata: attributes{
-				SSHKeys: []string{"name:ssh-rsa [KEY] project1", "othername:ssh-rsa [KEY] project2"},
+				SSHKeys: []string{
+					fmt.Sprintf("name:ssh-rsa %s project1", pubKey),
+					fmt.Sprintf("othername:ssh-rsa %s project2", pubKey),
+				},
+			},
+			expectedKeys: []string{
+				fmt.Sprintf("ssh-rsa %s instance1", pubKey),
+				fmt.Sprintf("ssh-rsa %s project1", pubKey),
 			},
-			expectedKeys: []string{"ssh-rsa [KEY] instance1", "ssh-rsa [KEY] project1"},
 		},
 		{
 			userName: "name",
-			instanceMetadata: attributes{BlockProjectSSHKeys: true,
-				SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"},
+			instanceMetadata: attributes{
+				BlockProjectSSHKeys: true,
+				SSHKeys: []string{
+					fmt.Sprintf("name:ssh-rsa %s instance1", pubKey),
+					fmt.Sprintf("othername:ssh-rsa %s instance2", pubKey),
+				},
 			},
 			projectMetadata: attributes{
-				SSHKeys: []string{"name:ssh-rsa [KEY] project1", "othername:ssh-rsa [KEY] project2"},
+				SSHKeys: []string{
+					fmt.Sprintf("name:ssh-rsa %s project1", pubKey),
+					fmt.Sprintf("othername:ssh-rsa %s project2", pubKey),
+				},
 			},
-			expectedKeys: []string{"ssh-rsa [KEY] instance1"},
+			expectedKeys: []string{fmt.Sprintf("ssh-rsa %s instance1", pubKey)},
 		},
 		{
 			userName: "name",
-			instanceMetadata: attributes{BlockProjectSSHKeys: false,
-				SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"},
+			instanceMetadata: attributes{
+				BlockProjectSSHKeys: false,
+				SSHKeys: []string{
+					fmt.Sprintf("name:ssh-rsa %s instance1", pubKey),
+					fmt.Sprintf("othername:ssh-rsa %s instance2", pubKey),
+				},
 			},
 			projectMetadata: attributes{
 				SSHKeys: nil,
 			},
-			expectedKeys: []string{"ssh-rsa [KEY] instance1"},
+			expectedKeys: []string{fmt.Sprintf("ssh-rsa %s instance1", pubKey)},
 		},
 		{
 			userName: "name",
-			instanceMetadata: attributes{BlockProjectSSHKeys: false,
-				SSHKeys: nil,
+			instanceMetadata: attributes{
+				BlockProjectSSHKeys: false,
+				SSHKeys:             nil,
 			},
 			projectMetadata: attributes{
-				SSHKeys: []string{"name:ssh-rsa [KEY] project1", "othername:ssh-rsa [KEY] project2"},
+				SSHKeys: []string{
+					fmt.Sprintf("name:ssh-rsa %s project1", pubKey),
+					fmt.Sprintf("othername:ssh-rsa %s project2", pubKey),
+				},
 			},
-			expectedKeys: []string{"ssh-rsa [KEY] project1"},
+			expectedKeys: []string{fmt.Sprintf("ssh-rsa %s project1", pubKey)},
 		},
 	}
 
 	for count, tt := range tests {
-		if got, want := getUserKeys(tt.userName, &tt.instanceMetadata, &tt.projectMetadata), tt.expectedKeys; !stringSliceEqual(got, want) {
-			t.Errorf("getUserKeys[%d] incorrect return: got %v, want %v", count, got, want)
-		}
+		t.Run(fmt.Sprintf("test-%d", count), func(t *testing.T) {
+			if got, want := getUserKeys(tt.userName, &tt.instanceMetadata, &tt.projectMetadata), tt.expectedKeys; !stringSliceEqual(got, want) {
+				t.Errorf("getUserKeys[%d] incorrect return: got %v, want %v", count, got, want)
+			}
+		})
 	}
 }
 
 func TestGetMetadataAttributes(t *testing.T) {
 	tests := []struct {
-		metadata  string
 		att       *attributes
 		expectErr bool
 	}{
 		{
-			metadata:  `{"enable-windows-ssh":"true","ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","block-project-ssh-keys":"false","other-metadata":"foo"}`,
 			att:       &attributes{EnableWindowsSSH: truebool, SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"}, BlockProjectSSHKeys: false},
 			expectErr: false,
 		},
 		{
-			metadata:  `{"enable-windows-ssh":"true","ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","block-project-ssh-keys":"true","other-metadata":"foo"}`,
 			att:       &attributes{EnableWindowsSSH: truebool, SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"}, BlockProjectSSHKeys: true},
 			expectErr: false,
 		},
 		{
-			metadata:  `{"ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","block-project-ssh-keys":"false","other-metadata":"foo"}`,
 			att:       &attributes{EnableWindowsSSH: nil, SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"}, BlockProjectSSHKeys: false},
 			expectErr: false,
 		},
 		{
-			metadata:  `{"enable-windows-ssh":"false","ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","other-metadata":"foo"}`,
 			att:       &attributes{EnableWindowsSSH: falsebool, SSHKeys: []string{"name:ssh-rsa [KEY] instance1", "othername:ssh-rsa [KEY] instance2"}, BlockProjectSSHKeys: false},
 			expectErr: false,
 		},
 		{
-			metadata:  `BADJSON`,
 			att:       nil,
 			expectErr: true,
 		},
 	}
 
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Get test number from request path
-		tnum, _ := strconv.Atoi(strings.Split(r.URL.Path, "/")[2])
-		fmt.Fprintf(w, tests[tnum].metadata)
-	}))
-
-	defer ts.Close()
-
-	metadataURL = ts.URL
-	defaultTimeout = 1 * time.Second
+	client = &mdsClient{}
 
 	for count, tt := range tests {
 		want := tt.att
 		hasErr := false
 		reqStr := fmt.Sprintf("/attributes/%d", count)
-		got, err := getMetadataAttributes(reqStr)
+		got, err := getMetadataAttributes(context.Background(), reqStr)
 		if err != nil {
 			hasErr = true
 		}
@@ -230,3 +249,43 @@ func TestGetMetadataAttributes(t *testing.T) {
 		}
 	}
 }
+
+type mdsClient struct{}
+
+func (mds *mdsClient) Get(ctx context.Context) (*metadata.Descriptor, error) {
+	return nil, fmt.Errorf("Get() not yet implemented")
+}
+
+func (mds *mdsClient) GetKey(ctx context.Context, key string, headers map[string]string) (string, error) {
+	return "", fmt.Errorf("GetKey() not yet implemented")
+}
+
+func (mds *mdsClient) GetKeyRecursive(ctx context.Context, key string) (string, error) {
+	i, err := strconv.Atoi(key[strings.LastIndex(key, "/")+1:])
+	if err != nil {
+		return "", err
+	}
+
+	switch i {
+	case 0:
+		return `{"enable-windows-ssh":"true","ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","block-project-ssh-keys":"false","other-metadata":"foo"}`, nil
+	case 1:
+		return `{"enable-windows-ssh":"true","ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","block-project-ssh-keys":"true","other-metadata":"foo"}`, nil
+	case 2:
+		return `{"ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","block-project-ssh-keys":"false","other-metadata":"foo"}`, nil
+	case 3:
+		return `{"enable-windows-ssh":"false","ssh-keys":"name:ssh-rsa [KEY] instance1\nothername:ssh-rsa [KEY] instance2","other-metadata":"foo"}`, nil
+	case 4:
+		return "BADJSON", nil
+	default:
+		return "", fmt.Errorf("unknown key %q", key)
+	}
+}
+
+func (mds *mdsClient) Watch(ctx context.Context) (*metadata.Descriptor, error) {
+	return nil, fmt.Errorf("Watch() not yet implemented")
+}
+
+func (mds *mdsClient) WriteGuestAttributes(ctx context.Context, key string, value string) error {
+	return fmt.Errorf("WriteGuestattributes() not yet implemented")
+}
diff --git a/google_guest_agent/addresses.go b/google_guest_agent/addresses.go
index 7c0c741..6f13797 100644
--- a/google_guest_agent/addresses.go
+++ b/google_guest_agent/addresses.go
@@ -19,25 +19,21 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"os"
 	"reflect"
 	"runtime"
+	"slices"
 	"strings"
-	"time"
 
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
+	network "github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/network/manager"
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/run"
-	"github.com/GoogleCloudPlatform/guest-agent/metadata"
-	"github.com/GoogleCloudPlatform/guest-agent/utils"
 	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
 )
 
 var (
-	addressKey        = regKeyBase + `\ForwardedIps`
-	oldWSFCAddresses  string
-	oldWSFCEnable     bool
-	interfacesEnabled bool
-	interfaces        []net.Interface
+	addressKey       = regKeyBase + `\ForwardedIps`
+	oldWSFCAddresses string
+	oldWSFCEnable    bool
 )
 
 type addressMgr struct{}
@@ -78,7 +74,9 @@ func getForwardsFromRegistry(mac string) ([]string, error) {
 		oldName := strings.Replace(mac, ":", "", -1)
 		regFwdIPs, err = readRegMultiString(addressKey, oldName)
 		if err == nil {
-			deleteRegKey(addressKey, oldName)
+			if err = deleteRegKey(addressKey, oldName); err != nil {
+				logger.Warningf("Failed to delete key: %q, name: %q from registry", addressKey, oldName)
+			}
 		}
 	} else if err != nil {
 		return nil, err
@@ -88,13 +86,13 @@ func getForwardsFromRegistry(mac string) ([]string, error) {
 
 func compareRoutes(configuredRoutes, desiredRoutes []string) (toAdd, toRm []string) {
 	for _, desiredRoute := range desiredRoutes {
-		if !utils.ContainsString(desiredRoute, configuredRoutes) {
+		if !slices.Contains(configuredRoutes, desiredRoute) {
 			toAdd = append(toAdd, desiredRoute)
 		}
 	}
 
 	for _, configuredRoute := range configuredRoutes {
-		if !utils.ContainsString(configuredRoute, desiredRoutes) {
+		if !slices.Contains(desiredRoutes, configuredRoute) {
 			toRm = append(toRm, configuredRoute)
 		}
 	}
@@ -103,20 +101,6 @@ func compareRoutes(configuredRoutes, desiredRoutes []string) (toAdd, toRm []stri
 
 var badMAC []string
 
-func getInterfaceByMAC(mac string) (net.Interface, error) {
-	hwaddr, err := net.ParseMAC(mac)
-	if err != nil {
-		return net.Interface{}, err
-	}
-
-	for _, iface := range interfaces {
-		if iface.HardwareAddr.String() == hwaddr.String() {
-			return iface, nil
-		}
-	}
-	return net.Interface{}, fmt.Errorf("no interface found with MAC %s", mac)
-}
-
 // https://www.ietf.org/rfc/rfc1354.txt
 // Only fields that we currently care about.
 type ipForwardEntry struct {
@@ -221,7 +205,7 @@ func (a *addressMgr) applyWSFCFilter(config *cfg.Sections) {
 		for idx := range interfaces {
 			var filteredForwardedIps []string
 			for _, ip := range interfaces[idx].ForwardedIps {
-				if !utils.ContainsString(ip, wsfcAddrs) {
+				if !slices.Contains(wsfcAddrs, ip) {
 					filteredForwardedIps = append(filteredForwardedIps, ip)
 				}
 			}
@@ -229,7 +213,7 @@ func (a *addressMgr) applyWSFCFilter(config *cfg.Sections) {
 
 			var filteredTargetInstanceIps []string
 			for _, ip := range interfaces[idx].TargetInstanceIps {
-				if !utils.ContainsString(ip, wsfcAddrs) {
+				if !slices.Contains(wsfcAddrs, ip) {
 					filteredTargetInstanceIps = append(filteredTargetInstanceIps, ip)
 				}
 			}
@@ -290,28 +274,10 @@ func (a *addressMgr) Set(ctx context.Context) error {
 		a.applyWSFCFilter(config)
 	}
 
-	var err error
-	interfaces, err = net.Interfaces()
+	// Setup network interfaces.
+	err := network.SetupInterfaces(ctx, config, newMetadata.Instance.NetworkInterfaces)
 	if err != nil {
-		return fmt.Errorf("error populating interfaces: %v", err)
-	}
-
-	if config.NetworkInterfaces.Setup {
-		if runtime.GOOS != "windows" {
-			logger.Debugf("Configure IPv6")
-			if err := configureIPv6(ctx); err != nil {
-				// Continue through IPv6 configuration errors.
-				logger.Errorf("Error configuring IPv6: %v", err)
-			}
-		}
-
-		if runtime.GOOS != "windows" && !interfacesEnabled {
-			logger.Debugf("Enable network interfaces")
-			if err := enableNetworkInterfaces(ctx, config); err != nil {
-				return err
-			}
-			interfacesEnabled = true
-		}
+		return fmt.Errorf("failed to setup network interfaces: %v", err)
 	}
 
 	if !config.NetworkInterfaces.IPForwarding {
@@ -321,9 +287,9 @@ func (a *addressMgr) Set(ctx context.Context) error {
 	logger.Debugf("Add routes for aliases, forwarded IP and target-instance IPs")
 	// Add routes for IP aliases, forwarded and target-instance IPs.
 	for _, ni := range newMetadata.Instance.NetworkInterfaces {
-		iface, err := getInterfaceByMAC(ni.Mac)
+		iface, err := network.GetInterfaceByMAC(ni.Mac)
 		if err != nil {
-			if !utils.ContainsString(ni.Mac, badMAC) {
+			if !slices.Contains(badMAC, ni.Mac) {
 				logger.Errorf("Error getting interface: %s", err)
 				badMAC = append(badMAC, ni.Mac)
 			}
@@ -356,7 +322,7 @@ func (a *addressMgr) Set(ctx context.Context) error {
 			}
 			for _, ip := range configuredIPs {
 				// Only add to `forwardedIPs` if it is recorded in the registry.
-				if utils.ContainsString(ip, regFwdIPs) {
+				if slices.Contains(regFwdIPs, ip) {
 					forwardedIPs = append(forwardedIPs, ip)
 				}
 			}
@@ -399,14 +365,14 @@ func (a *addressMgr) Set(ctx context.Context) error {
 		var registryEntries []string
 		for _, ip := range wantIPs {
 			// If the IP is not in toAdd, add to registry list and continue.
-			if !utils.ContainsString(ip, toAdd) {
+			if !slices.Contains(toAdd, ip) {
 				registryEntries = append(registryEntries, ip)
 				continue
 			}
 			var err error
 			if runtime.GOOS == "windows" {
 				// Don't addAddress if this is already configured.
-				if !utils.ContainsString(ip, configuredIPs) {
+				if !slices.Contains(configuredIPs, ip) {
 					err = addAddress(net.ParseIP(ip), net.IPv4Mask(255, 255, 255, 255), uint32(iface.Index))
 				}
 			} else {
@@ -422,7 +388,7 @@ func (a *addressMgr) Set(ctx context.Context) error {
 		for _, ip := range toRm {
 			var err error
 			if runtime.GOOS == "windows" {
-				if !utils.ContainsString(ip, configuredIPs) {
+				if !slices.Contains(configuredIPs, ip) {
 					continue
 				}
 				err = removeAddress(net.ParseIP(ip), uint32(iface.Index))
@@ -445,193 +411,3 @@ func (a *addressMgr) Set(ctx context.Context) error {
 
 	return nil
 }
-
-// Enables or disables IPv6 on network interfaces.
-func configureIPv6(ctx context.Context) error {
-	var newNi, oldNi metadata.NetworkInterfaces
-	if len(newMetadata.Instance.NetworkInterfaces) == 0 {
-		return fmt.Errorf("no interfaces found in metadata")
-	}
-	newNi = newMetadata.Instance.NetworkInterfaces[0]
-	if len(oldMetadata.Instance.NetworkInterfaces) > 0 {
-		oldNi = oldMetadata.Instance.NetworkInterfaces[0]
-	}
-	iface, err := getInterfaceByMAC(newNi.Mac)
-	if err != nil {
-		return err
-	}
-	switch {
-	case oldNi.DHCPv6Refresh != "" && newNi.DHCPv6Refresh == "",
-		newNi.DHCPv6Refresh == "" && len(oldMetadata.Instance.NetworkInterfaces) == 0:
-		// disable
-		// uses empty old interface slice to indicate this is first-run.
-
-		// Before obtaining or releasing an IPv6 lease, we wait for
-		// 'tentative' IPs as part of SLAAC. We wait up to 5 seconds
-		// for this condition to automatically resolve.
-		tentative := []string{"-6", "-o", "a", "s", "dev", iface.Name, "scope", "link", "tentative"}
-		for i := 0; i < 5; i++ {
-			res := run.WithOutput(ctx, "ip", tentative...)
-			if res.ExitCode == 0 && res.StdOut == "" {
-				break
-			}
-			time.Sleep(1 * time.Second)
-		}
-		if err := run.Quiet(ctx, "dhclient", "-r", "-6", "-1", "-v", iface.Name); err != nil {
-			return err
-		}
-	case oldNi.DHCPv6Refresh == "" && newNi.DHCPv6Refresh != "":
-		// enable
-		tentative := []string{"-6", "-o", "a", "s", "dev", iface.Name, "scope", "link", "tentative"}
-		for i := 0; i < 5; i++ {
-			res := run.WithOutput(ctx, "ip", tentative...)
-			if res.ExitCode == 0 && res.StdOut == "" {
-				break
-			}
-			time.Sleep(1 * time.Second)
-		}
-		val := fmt.Sprintf("net.ipv6.conf.%s.accept_ra_rt_info_max_plen=128", iface.Name)
-		if err := run.Quiet(ctx, "sysctl", val); err != nil {
-			return err
-		}
-		if err := run.Quiet(ctx, "dhclient", "-1", "-6", "-v", iface.Name); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// enableNetworkInterfaces runs `dhclient eth1 eth2 ... ethN`
-// and `dhclient -6 eth1 eth2 ... ethN`.
-// On RHEL7, it also calls disableNM for each interface.
-// On SLES, it calls enableSLESInterfaces instead of dhclient.
-func enableNetworkInterfaces(ctx context.Context, config *cfg.Sections) error {
-	if len(newMetadata.Instance.NetworkInterfaces) < 2 {
-		return nil
-	}
-	var googleInterfaces []string
-	// The primary (first) interface is managed by the OS, we only handle
-	// secondary interfaces in this code.
-	for _, ni := range newMetadata.Instance.NetworkInterfaces[1:] {
-		iface, err := getInterfaceByMAC(ni.Mac)
-		if err != nil {
-			if !utils.ContainsString(ni.Mac, badMAC) {
-				logger.Errorf("Error getting interface: %s", err)
-				badMAC = append(badMAC, ni.Mac)
-			}
-			continue
-		}
-		googleInterfaces = append(googleInterfaces, iface.Name)
-	}
-	var googleIpv6Interfaces []string
-	for _, ni := range newMetadata.Instance.NetworkInterfaces[1:] {
-		if ni.DHCPv6Refresh == "" {
-			// This interface is not IPv6 enabled
-			continue
-		}
-		iface, err := getInterfaceByMAC(ni.Mac)
-		if err != nil {
-			if !utils.ContainsString(ni.Mac, badMAC) {
-				logger.Errorf("Error getting interface: %s", err)
-				badMAC = append(badMAC, ni.Mac)
-			}
-			continue
-		}
-		googleIpv6Interfaces = append(googleIpv6Interfaces, iface.Name)
-	}
-
-	switch {
-	case osInfo.OS == "sles":
-		return enableSLESInterfaces(ctx, googleInterfaces)
-	case (osInfo.OS == "rhel" || osInfo.OS == "centos") && osInfo.Version.Major >= 7:
-		for _, iface := range googleInterfaces {
-			err := disableNM(iface)
-			if err != nil {
-				return err
-			}
-		}
-		fallthrough
-	default:
-		dhcpCommand := config.NetworkInterfaces.DHCPCommand
-		if dhcpCommand != "" {
-			tokens := strings.Split(dhcpCommand, " ")
-			return run.Quiet(ctx, tokens[0], tokens[1:]...)
-		}
-
-		// Try IPv4 first as it's higher priority.
-		if err := run.Quiet(ctx, "dhclient", googleInterfaces...); err != nil {
-			return err
-		}
-
-		if len(googleIpv6Interfaces) == 0 {
-			return nil
-		}
-		for _, iface := range googleIpv6Interfaces {
-			// Enable kernel to accept to route advertisements.
-			val := fmt.Sprintf("net.ipv6.conf.%s.accept_ra_rt_info_max_plen=128", iface)
-			if err := run.Quiet(ctx, "sysctl", val); err != nil {
-				return err
-			}
-		}
-
-		var dhclientArgs6 []string
-		dhclientArgs6 = append([]string{"-6"}, googleIpv6Interfaces...)
-		return run.Quiet(ctx, "dhclient", dhclientArgs6...)
-	}
-}
-
-// enableSLESInterfaces writes one ifcfg file for each interface, then
-// runs `wicked ifup eth1 eth2 ... ethN`
-func enableSLESInterfaces(ctx context.Context, interfaces []string) error {
-	var err error
-	var priority = 10100
-	for _, iface := range interfaces {
-		logger.Debugf("write enabling ifcfg-%s config", iface)
-
-		var ifcfg *os.File
-		ifcfg, err = os.Create("/etc/sysconfig/network/ifcfg-" + iface)
-		if err != nil {
-			return err
-		}
-		defer closer(ifcfg)
-		contents := []string{
-			googleComment,
-			"STARTMODE=hotplug",
-			// NOTE: 'dhcp' is the dhcp4+dhcp6 option.
-			"BOOTPROTO=dhcp",
-			fmt.Sprintf("DHCLIENT_ROUTE_PRIORITY=%d", priority),
-		}
-		_, err = ifcfg.WriteString(strings.Join(contents, "\n"))
-		if err != nil {
-			return err
-		}
-		priority += 100
-	}
-	args := append([]string{"ifup", "--timeout", "1"}, interfaces...)
-	return run.Quiet(ctx, "/usr/sbin/wicked", args...)
-}
-
-// disableNM writes an ifcfg file with DHCP and NetworkManager disabled.
-func disableNM(iface string) error {
-	logger.Debugf("write disabling ifcfg-%s config", iface)
-	filename := "/etc/sysconfig/network-scripts/ifcfg-" + iface
-	ifcfg, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
-	if err == nil {
-		defer closer(ifcfg)
-		contents := []string{
-			googleComment,
-			fmt.Sprintf("DEVICE=%s", iface),
-			"BOOTPROTO=none",
-			"DEFROUTE=no",
-			"IPV6INIT=no",
-			"NM_CONTROLLED=no",
-			"NOZEROCONF=yes",
-		}
-		_, err = ifcfg.WriteString(strings.Join(contents, "\n"))
-		return err
-	}
-	if os.IsExist(err) {
-		return nil
-	}
-	return err
-}
diff --git a/google_guest_agent/addresses_integ_test.go b/google_guest_agent/addresses_integ_test.go
deleted file mode 100644
index 1486986..0000000
--- a/google_guest_agent/addresses_integ_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2021 Google LLC
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-
-//     https://www.apache.org/licenses/LICENSE-2.0
-
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build integration
-// +build integration
-
-package main
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"testing"
-)
-
-const testIp = "192.168.0.0"
-
-func TestAddAndRemoveLocalRoute(t *testing.T) {
-	metdata, err := getMetadata(context.Context(), false)
-	if err != nil {
-		t.Fatalf("failed to get metadata, err %v", err)
-	}
-	iface, err := getInterfaceByMAC(metdata.Instance.NetworkInterfaces[0].Mac)
-	if err != nil {
-		t.Fatalf("failed to get interface from mac, err %v", err)
-	}
-	// test add local route
-	if err := removeLocalRoute(testIp, iface.Name); err != nil {
-		t.Fatalf("failed to remove local route, err %v", err)
-	}
-	if err := addLocalRoute(testIp, iface.Name); err != nil {
-		t.Fatalf("add test local route should not failed, err %v", err)
-	}
-
-	res, err := getLocalRoutes(iface.Name)
-	if err != nil {
-		t.Fatalf("get local route should not failed, err %v", err)
-	}
-	exist := false
-	for _, route := range res {
-		if strings.Contains(route, fmt.Sprintf("local %s/24", testIp)) {
-			exist = true
-		}
-	}
-	if !exist {
-		t.Fatalf("route %s is not added", testIp)
-	}
-
-	// test remove local route
-	if err := removeLocalRoute(testIp, iface.Name); err != nil {
-		t.Fatalf("add test local route should not failed")
-	}
-	res, err := getLocalRoutes(iface.Name)
-	if err != nil {
-		t.Fatalf("ip route list should not failed, err %s", res.err)
-	}
-
-	for _, route := range res {
-		if strings.Contains(route, fmt.Sprintf("local %s/24", testIp)) {
-			t.Fatalf("route %s should be removed but exist", testIp)
-		}
-	}
-}
-
-func TestGetLocalRoute(t *testing.T) {
-	metdata, err := getMetadata(context.Context(), false)
-	if err != nil {
-		t.Fatalf("failed to get metadata, err %v", err)
-	}
-	iface, err := getInterfaceByMAC(metdata.Instance.NetworkInterfaces[0].Mac)
-	if err != nil {
-		t.Fatalf("failed to get interface from mac, err %v", err)
-	}
-
-	if err := addLocalRoute(testIp, iface.Name); err != nil {
-		t.Fatalf("add test local route should not failed, err %v", err)
-	}
-	routes, err := getLocalRoutes(iface.Name)
-	if err != nil {
-		t.Fatalf("get local routes should not failed, err %v", err)
-	}
-	if len(routes) != 1 {
-		t.Fatal("find unexpected local route %s.", routes[0])
-	}
-	if routes[0] != testIp {
-		t.Fatal("find unexpected local route %s.", routes[0])
-	}
-}
diff --git a/google_guest_agent/agentcrypto/mtls_mds.go b/google_guest_agent/agentcrypto/mtls_mds.go
index ad61634..ca3dfe1 100644
--- a/google_guest_agent/agentcrypto/mtls_mds.go
+++ b/google_guest_agent/agentcrypto/mtls_mds.go
@@ -35,7 +35,7 @@ import (
 const (
 	// UEFI variables are of format {VariableName}-{VendorGUID}
 	// googleGUID is Google's (vendors/variable owners) GUID used to prevent name collision with other vendors.
-	googleGUID = "8be4df61-93ca-11d2-aa0d-00e098032b8c"
+	googleGUID = "a2858e46-a37f-456a-8c79-0c1fe48b65ff"
 	// googleRootCACertEFIVarName is predefined string part of the UEFI variable name that holds Root CA cert.
 	googleRootCACertEFIVarName = "InstanceRootCACertificate"
 	// clientCertsKey is the metadata server key at which client identity certificate is exposed.
diff --git a/google_guest_agent/agentcrypto/mtls_mds_linux.go b/google_guest_agent/agentcrypto/mtls_mds_linux.go
index ee64ff1..3a75ff0 100644
--- a/google_guest_agent/agentcrypto/mtls_mds_linux.go
+++ b/google_guest_agent/agentcrypto/mtls_mds_linux.go
@@ -17,6 +17,7 @@ package agentcrypto
 import (
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
 	"path/filepath"
 
@@ -35,8 +36,25 @@ const (
 	clientCredsFileName = "client.key"
 )
 
+var (
+	// certUpdaters is a map of known CA certificate updaters with the local directory paths for certificates.
+	certUpdaters = map[string][]string{
+		// SUSE, Debian and Ubuntu distributions.
+		// https://manpages.ubuntu.com/manpages/xenial/man8/update-ca-certificates.8.html
+		// https://github.com/openSUSE/ca-certificates
+		"update-ca-certificates": {"/usr/local/share/ca-certificates", "/usr/share/pki/trust/anchors"},
+		// CentOS, Fedora, RedHat distributions.
+		// https://www.unix.com/man-page/centos/8/UPDATE-CA-TRUST
+		"update-ca-trust": {"/etc/pki/ca-trust/source/anchors"},
+	}
+)
+
 // writeRootCACert writes Root CA cert from UEFI variable to output file.
 func (j *CredsJob) writeRootCACert(ctx context.Context, content []byte, outputFile string) error {
+	// The directory should be executable, but the file does not need to be.
+	if err := os.MkdirAll(filepath.Dir(outputFile), 0655); err != nil {
+		return err
+	}
 	if err := utils.SaferWriteFile(content, outputFile, 0644); err != nil {
 		return err
 	}
@@ -51,39 +69,42 @@ func (j *CredsJob) writeRootCACert(ctx context.Context, content []byte, outputFi
 
 // writeClientCredentials stores client credentials (certificate and private key).
 func (j *CredsJob) writeClientCredentials(plaintext []byte, outputFile string) error {
+	// The directory should be executable, but the file does not need to be.
+	if err := os.MkdirAll(filepath.Dir(outputFile), 0655); err != nil {
+		return err
+	}
 	return utils.SaferWriteFile(plaintext, outputFile, 0644)
 }
 
 // getCAStoreUpdater interates over known system trust store updaters and returns the first found.
 func getCAStoreUpdater() (string, error) {
-	knownUpdaters := []string{"update-ca-certificates", "update-ca-trust"}
 	var errs []string
 
-	for _, u := range knownUpdaters {
+	for u := range certUpdaters {
 		_, err := exec.LookPath(u)
 		if err == nil {
 			return u, nil
 		}
-		errs = append(errs, err.Error())
+		errs = append(errs, fmt.Sprintf("lookup for %q failed with error: %v", u, err))
 	}
 
-	return "", fmt.Errorf("no known trust updaters %v were found: %v", knownUpdaters, errs)
+	return "", fmt.Errorf("no known trust updaters were found: %v", errs)
 }
 
 // certificateDirFromUpdater returns directory of local CA certificates for the given updater tool.
 func certificateDirFromUpdater(updater string) (string, error) {
-	switch updater {
-	// SUSE, Debian and Ubuntu distributions.
-	// https://manpages.ubuntu.com/manpages/xenial/man8/update-ca-certificates.8.html
-	case "update-ca-certificates":
-		return "/usr/local/share/ca-certificates/", nil
-	// CentOS, Fedora, RedHat distributions.
-	// https://www.unix.com/man-page/centos/8/UPDATE-CA-TRUST/
-	case "update-ca-trust":
-		return "/etc/pki/ca-trust/source/anchors/", nil
-	default:
+	dirs, ok := certUpdaters[updater]
+	if !ok {
 		return "", fmt.Errorf("unknown updater %q, no local trusted CA certificate directory found", updater)
 	}
+
+	for _, dir := range dirs {
+		fi, err := os.Stat(dir)
+		if err == nil && fi.IsDir() {
+			return dir, nil
+		}
+	}
+	return "", fmt.Errorf("no of the known directories %v found for updater %q", dirs, updater)
 }
 
 // updateSystemStore updates the local system store with the cert.
diff --git a/google_guest_agent/agentcrypto/mtls_mds_linux_test.go b/google_guest_agent/agentcrypto/mtls_mds_linux_test.go
index 020af48..831e02f 100644
--- a/google_guest_agent/agentcrypto/mtls_mds_linux_test.go
+++ b/google_guest_agent/agentcrypto/mtls_mds_linux_test.go
@@ -132,17 +132,24 @@ func TestShouldEnableError(t *testing.T) {
 }
 
 func TestCertificateDirFromUpdater(t *testing.T) {
+	updater1Dir := t.TempDir()
+	updater2Dir := t.TempDir()
+	certUpdaters = map[string][]string{
+		"updater1": {updater1Dir},
+		"updater2": {"/does/not/exist", updater2Dir},
+	}
+
 	tests := []struct {
 		updater string
 		want    string
 	}{
 		{
-			updater: "update-ca-certificates",
-			want:    "/usr/local/share/ca-certificates/",
+			updater: "updater1",
+			want:    updater1Dir,
 		},
 		{
-			updater: "update-ca-trust",
-			want:    "/etc/pki/ca-trust/source/anchors/",
+			updater: "updater2",
+			want:    updater2Dir,
 		},
 	}
 
@@ -160,8 +167,18 @@ func TestCertificateDirFromUpdater(t *testing.T) {
 }
 
 func TestCertificateDirFromUpdaterError(t *testing.T) {
+	// Fail for unknown updater.
 	_, err := certificateDirFromUpdater("unknown")
 	if err == nil {
 		t.Errorf("certificateDirFromUpdater(unknown) succeeded for unknown updater, want error")
 	}
+
+	// Fail for missing known cert dir.
+	certUpdaters = map[string][]string{
+		"updater1": {"/no/dir/exist"},
+	}
+	_, err = certificateDirFromUpdater("updater1")
+	if err == nil {
+		t.Errorf("certificateDirFromUpdater(unknown) succeeded for missing cert dir, want error")
+	}
 }
diff --git a/google_guest_agent/agentcrypto/mtls_mds_windows.go b/google_guest_agent/agentcrypto/mtls_mds_windows.go
index a11f113..3f70954 100644
--- a/google_guest_agent/agentcrypto/mtls_mds_windows.go
+++ b/google_guest_agent/agentcrypto/mtls_mds_windows.go
@@ -59,6 +59,12 @@ var (
 
 // writeRootCACert writes Root CA cert from UEFI variable to output file.
 func (j *CredsJob) writeRootCACert(_ context.Context, cacert []byte, outputFile string) error {
+	// Try to fetch previous certificate's serial number before it gets overwritten.
+	num, err := serialNumber(outputFile)
+	if err != nil {
+		logger.Debugf("No previous MDS root certificate was found, will skip cleanup: %v", err)
+	}
+
 	if err := utils.SaferWriteFile(cacert, outputFile, 0644); err != nil {
 		return err
 	}
@@ -84,6 +90,25 @@ func (j *CredsJob) writeRootCACert(_ context.Context, cacert []byte, outputFile 
 		return fmt.Errorf("failed to store root cert ctx in store: %w", err)
 	}
 
+	// MDS root cert was not refreshed or there's no previous cert, nothing to do, return.
+	if num == "" || fmt.Sprintf("%x", x509Cert.SerialNumber) == num {
+		return nil
+	}
+
+	// Certificate is refreshed. Best effort to find the certcontext and delete it.
+	// Don't throw error here, it would skip client credential generation which
+	// may be about to expire.
+	oldCtx, err := findCert(root, certificateIssuer, num)
+	if err != nil {
+		logger.Warningf("Failed to find previous MDS root certificate with error: %v", err)
+		return nil
+	}
+
+	if err := deleteCert(oldCtx, root); err != nil {
+		logger.Warningf("Failed to delete previous MDS root certificate(%s) with error: %v", num, err)
+		return nil
+	}
+
 	return nil
 }
 
diff --git a/google_guest_agent/cfg/cfg.go b/google_guest_agent/cfg/cfg.go
index 3a54c47..c59ec14 100644
--- a/google_guest_agent/cfg/cfg.go
+++ b/google_guest_agent/cfg/cfg.go
@@ -27,6 +27,10 @@ var (
 	// should always return it.
 	instance *Sections
 
+	// configFile is a pointer to a function which takes the current OS name and returns
+	// an appropriate config file name. Replaceable by unit tests.
+	configFile = defaultConfigFile
+
 	// dataSource is a pointer to a data source loading/defining function, unit tests will
 	// want to change this pointer to whatever makes sense to its implementation.
 	dataSources = defaultDataSources
@@ -87,6 +91,9 @@ setup = true
 [OSLogin]
 cert_authentication = true
 
+[MDS]
+mtls_bootstrapping_enabled = true
+
 [Snapshots]
 enabled = false
 snapshot_service_ip = 169.254.169.254
@@ -94,7 +101,10 @@ snapshot_service_port = 8081
 timeout_in_seconds = 60
 
 [Unstable]
-mds_mtls = false
+command_monitor_enabled = false
+command_pipe_mode = 0770
+command_pipe_group =
+command_request_timeout = 10s
 `
 )
 
@@ -144,6 +154,9 @@ type Sections struct {
 	// OSLogin defines the OS Login configuration options.
 	OSLogin *OSLogin `ini:"OSLogin,omitempty"`
 
+	// MDS defines the MDS configuration options.
+	MDS *MDS `ini:"MDS,omitempty"`
+
 	// Snpashots defines the snapshot listener configuration and behavior i.e. the server address and port.
 	Snapshots *Snapshots `ini:"Snapshots,omitempty"`
 
@@ -237,6 +250,12 @@ type OSLogin struct {
 	CertAuthentication bool `ini:"cert_authentication,omitempty"`
 }
 
+// MDS contains the configurations for MDS section.
+type MDS struct {
+	// MTLSBootstrappingEnabled enables/disables the mTLS credential refresher.
+	MTLSBootstrappingEnabled bool `ini:"mtls_bootstrapping_enabled,omitempty"`
+}
+
 // NetworkInterfaces contains the configurations of NetworkInterfaces section.
 type NetworkInterfaces struct {
 	DHCPCommand  string `ini:"dhcp_command,omitempty"`
@@ -256,7 +275,11 @@ type Snapshots struct {
 // is guaranteed for configurations defined in the Unstable section. By default all flags defined
 // in this section is disabled and is intended to isolate under development features.
 type Unstable struct {
-	MDSMTLS bool `ini:"mds_mtls,omitempty"`
+	CommandMonitorEnabled bool   `ini:"command_monitor_enabled,omitempty"`
+	CommandPipePath       string `ini:"command_pipe_path,omitempty"`
+	CommandRequestTimeout string `ini:"command_request_timeout,omitempty"`
+	CommandPipeMode       string `ini:"command_pipe_mode,omitempty"`
+	CommandPipeGroup      string `ini:"command_pipe_group,omitempty"`
 }
 
 // WSFC contains the configurations of WSFC section.
@@ -274,18 +297,17 @@ func defaultConfigFile(osName string) string {
 }
 
 func defaultDataSources(extraDefaults []byte) []interface{} {
-	var res []interface{}
-	configFile := defaultConfigFile(runtime.GOOS)
+	var res = []interface{}{[]byte(defaultConfig)}
+	config := configFile(runtime.GOOS)
 
 	if len(extraDefaults) > 0 {
 		res = append(res, extraDefaults)
 	}
 
 	return append(res, []interface{}{
-		[]byte(defaultConfig),
-		configFile,
-		configFile + ".distro",
-		configFile + ".template",
+		config,
+		config + ".distro",
+		config + ".template",
 	}...)
 }
 
diff --git a/google_guest_agent/cfg/cfg_test.go b/google_guest_agent/cfg/cfg_test.go
index efea06f..20321c1 100644
--- a/google_guest_agent/cfg/cfg_test.go
+++ b/google_guest_agent/cfg/cfg_test.go
@@ -14,7 +14,9 @@
 
 package cfg
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestLoad(t *testing.T) {
 	if err := Load(nil); err != nil {
diff --git a/google_guest_agent/command/Readme.md b/google_guest_agent/command/Readme.md
new file mode 100644
index 0000000..4fabd96
--- /dev/null
+++ b/google_guest_agent/command/Readme.md
@@ -0,0 +1,24 @@
+# Guest Agent Command Monitor
+## Overview
+The Guest Agent command monitor is a system used for executing commands in the guest agent on behalf of components in the guest os.
+
+The events layer is formed of a **Monitor**, a **Server** and a **Handler** where the **Monitor** handles command registration for guest agent components, the **Server** is the component which listens for events from the gueest os, and the **Handler** is the function executed by the agent.
+
+Each **Handler** is identified by a string ID, provided when sending commands to the server. Requests and response to and from the server are structured in JSON format. A request must contain the name field, specifying the handler to be executed. A request may contain arbitrary other fields to be passed to the handler. An example request is below:
+
+```
+{"Name":"agent.ExampleCommand","ArbitraryArgument":123}
+```
+
+A response will be valid JSON and has two required fields: Status and StatusMessage. Status is an int which follows unix status code conventions (ie zero is success, status codes are arbitrary and meaning is defined by the function called) and StatusMessage is an explanatory string accompanying the Status. Two example responses are below.
+
+```
+{"Status":0,"StatusMessage":""}
+
+{"Status":7,"StatusMessage":"Failure message"}
+```
+
+By default, the Server listens on a unix socket or a named pipe, depending on platform. Permissions for the pipe and the pipe path can be set in the guest-agent [configuration](https://github.com/GoogleCloudPlatform/guest-agent#configuration). The default pipe path for windows and linux systems are `\\.\pipe\google-guest-agent-commands` non-windows and `/run/google-guest-agent/commands.sock` respectively.
+
+## Implementing a command handler
+Registering a command handler will expose the handler function to be called by anyone with write permission to the underlying socket. To do so, call `command.Get().RegisterHandler(name, handerFunc)` to get the current command monitor and register the handlerFunc with it. Note that if the command system is disabled by user configuration, handler registration will succeed but the server will not be available for callers to send commands to.
diff --git a/google_guest_agent/command/command.go b/google_guest_agent/command/command.go
new file mode 100644
index 0000000..527d20f
--- /dev/null
+++ b/google_guest_agent/command/command.go
@@ -0,0 +1,146 @@
+//  Copyright 2023 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// Package command facilitates calling commands within the guest-agent.
+package command
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
+)
+
+// Get returns the current command monitor which can be used to register command handlers.
+func Get() *Monitor {
+	return cmdMonitor
+}
+
+// Handler functions are the business logic of commands. They must process json
+// encoded as a byte slice which contains a Command field and optional arbitrary
+// data, and return json which contains a Status, StatusMessage, and optional
+// arbitrary data (again encoded as a byte slice). Returned errors will be
+// passed onto the command requester.
+type Handler func([]byte) ([]byte, error)
+
+// Request is the basic request structure. Command determines which handler the
+// request is routed to. Callers may set additional arbitrary fields.
+type Request struct {
+	Command string
+}
+
+// Response is the basic response structure. Handlers may set additional
+// arbitrary fields.
+type Response struct {
+	// Status code for the request. Meaning is defined by the caller, but
+	// conventially zero is success.
+	Status int
+	// StatusMessage is an optional message defined by the caller. Should generally
+	// help a human understand what happened.
+	StatusMessage string
+}
+
+var (
+	// CmdNotFoundError is return when there is no handler for the request command
+	CmdNotFoundError = Response{
+		Status:        101,
+		StatusMessage: "Could not find a handler for the requested command",
+	}
+	// BadRequestError is returned for invalid or unparseable JSON
+	BadRequestError = Response{
+		Status:        102,
+		StatusMessage: "Could not parse valid JSON from request",
+	}
+	// ConnError is returned for errors from the underlying communication protocol
+	ConnError = Response{
+		Status:        103,
+		StatusMessage: "Connection error",
+	}
+	// TimeoutError is returned when the timeout period elapses before valid JSON is receieved
+	TimeoutError = Response{
+		Status:        104,
+		StatusMessage: "Connection timeout before reading valid request",
+	}
+	// HandlerError is returned when the handler function returns an non-nil error. The status message will be replaced with the returnd error string.
+	HandlerError = Response{
+		Status:        105,
+		StatusMessage: "The command handler encountered an error processing your request",
+	}
+	// InternalErrorCode is the error code for internal command server errors. Returned when failing to marshal a response.
+	InternalErrorCode = 106
+	internalError     = []byte(`{"Status":106,"StatusMessage":"The command server encountered an internal error trying to respond to your request"}`)
+)
+
+// RegisterHandler registers f as the handler for cmd. If a command.Server has
+// been initialized, it will be signalled to start listening for commands.
+func (m *Monitor) RegisterHandler(cmd string, f Handler) error {
+	m.handlersMu.Lock()
+	defer m.handlersMu.Unlock()
+	if _, ok := m.handlers[cmd]; ok {
+		return fmt.Errorf("cmd %s is already handled", cmd)
+	}
+	m.handlers[cmd] = f
+	return nil
+}
+
+// UnregisterHandler clears the handlers for cmd. If a command.Server has been
+// intialized and there are no more handlers registered, the server will be
+// signalled to stop listening for commands.
+func (m *Monitor) UnregisterHandler(cmd string) error {
+	m.handlersMu.Lock()
+	defer m.handlersMu.Unlock()
+	if _, ok := m.handlers[cmd]; !ok {
+		return fmt.Errorf("cmd %s is not registered", cmd)
+	}
+	delete(m.handlers, cmd)
+	return nil
+}
+
+// SendCommand sends a command request over the configured pipe.
+func SendCommand(ctx context.Context, req []byte) []byte {
+	pipe := cfg.Get().Unstable.CommandPipePath
+	if pipe == "" {
+		pipe = DefaultPipePath
+	}
+	return SendCmdPipe(ctx, pipe, req)
+}
+
+// SendCmdPipe sends a command request over a specific pipe. Most callers
+// should use SendCommand() instead.
+func SendCmdPipe(ctx context.Context, pipe string, req []byte) []byte {
+	conn, err := dialPipe(ctx, pipe)
+	if err != nil {
+		if b, err := json.Marshal(ConnError); err != nil {
+			return b
+		}
+		return internalError
+	}
+	i, err := conn.Write(req)
+	if err != nil || i != len(req) {
+		if b, err := json.Marshal(ConnError); err != nil {
+			return b
+		}
+		return internalError
+	}
+	data, err := io.ReadAll(conn)
+	if err != nil {
+		if b, err := json.Marshal(ConnError); err != nil {
+			return b
+		}
+		return internalError
+	}
+	return data
+}
diff --git a/google_guest_agent/command/command_linux.go b/google_guest_agent/command/command_linux.go
new file mode 100644
index 0000000..cf7dab1
--- /dev/null
+++ b/google_guest_agent/command/command_linux.go
@@ -0,0 +1,140 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/user"
+	"path"
+	"runtime"
+	"strconv"
+	"syscall"
+
+	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
+)
+
+// DefaultPipePath is the default unix socket path for linux.
+const DefaultPipePath = "/run/google-guest-agent/commands.sock"
+
+func mkdirpWithPerms(dir string, p os.FileMode, uid, gid int) error {
+	stat, err := os.Stat(dir)
+	if err == nil {
+		statT, ok := stat.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("could not determine owner of %s", dir)
+		}
+		if !stat.IsDir() {
+			return fmt.Errorf("%s exists and is not a directory", dir)
+		}
+		if morePermissive(int(stat.Mode()), int(p)) {
+			if err := os.Chmod(dir, p); err != nil {
+				return fmt.Errorf("could not correct %s permissions to %d: %v", dir, p, err)
+			}
+		}
+		if statT.Uid != 0 && statT.Uid != uint32(uid) {
+			if err := os.Chown(dir, uid, -1); err != nil {
+				return fmt.Errorf("could not correct %s owner to %d: %v", dir, uid, err)
+			}
+		}
+		if statT.Gid != 0 && statT.Gid != uint32(gid) {
+			if err := os.Chown(dir, -1, gid); err != nil {
+				return fmt.Errorf("could not correct %s group to %d: %v", dir, gid, err)
+			}
+		}
+	} else {
+		parent, _ := path.Split(dir)
+		if parent != "/" && parent != "" {
+			if err := mkdirpWithPerms(parent, p, uid, gid); err != nil {
+				return err
+			}
+		}
+		if err := os.Mkdir(dir, p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func morePermissive(i, j int) bool {
+	for k := 0; k < 3; k++ {
+		if (i % 010) > (j % 10) {
+			return true
+		}
+		i = i / 010
+		j = j / 010
+	}
+	return false
+}
+
+func listen(ctx context.Context, pipe string, filemode int, grp string) (net.Listener, error) {
+	// If grp is an int, use it as a GID
+	gid, err := strconv.Atoi(grp)
+	if err != nil {
+		// Otherwise lookup GID
+		group, err := user.LookupGroup(grp)
+		if err != nil {
+			logger.Errorf("guest agent command pipe group %s is not a GID nor a valid group, not changing socket ownership", grp)
+			gid = -1
+		} else {
+			gid, err = strconv.Atoi(group.Gid)
+			if err != nil {
+				logger.Errorf("os reported group %s has gid %s which is not a valid int, not changing socket ownership. this should never happen", grp, group.Gid)
+				gid = -1
+			}
+		}
+	}
+	// socket owner group does not need to have permissions to everything in the directory containing it, whatever user and group we are should own that
+	user, err := user.Current()
+	if err != nil {
+		return nil, fmt.Errorf("could not lookup current user")
+	}
+	currentuid, err := strconv.Atoi(user.Uid)
+	if err != nil {
+		return nil, fmt.Errorf("os reported user %s has uid %s which is not a valid int, can't determine directory owner. this should never happen", user.Username, user.Uid)
+	}
+	currentgid, err := strconv.Atoi(user.Gid)
+	if err != nil {
+		return nil, fmt.Errorf("os reported user %s has gid %s which is not a valid int, can't determine directory owner. this should never happen", user.Username, user.Gid)
+	}
+	if err := mkdirpWithPerms(path.Dir(pipe), os.FileMode(filemode), currentuid, currentgid); err != nil {
+		return nil, err
+	}
+	// Mutating the umask of the process for this is not ideal, but tightening permissions with chown after creation is not really secure.
+	// Lock OS thread while mutating umask so we don't lose a thread with a mutated mask.
+	runtime.LockOSThread()
+	oldmask := syscall.Umask(777 - filemode)
+	var lc net.ListenConfig
+	l, err := lc.Listen(ctx, "unix", pipe)
+	syscall.Umask(oldmask)
+	runtime.UnlockOSThread()
+	if err != nil {
+		return nil, err
+	}
+	// But we need to chown anyway to loosen permissions to include whatever group the user has configured
+	err = os.Chown(pipe, int(currentuid), gid)
+	if err != nil {
+		l.Close()
+		return nil, err
+	}
+	return l, nil
+}
+
+func dialPipe(ctx context.Context, pipe string) (net.Conn, error) {
+	var dialer net.Dialer
+	return dialer.DialContext(ctx, "unix", pipe)
+}
diff --git a/google_guest_agent/command/command_monitor.go b/google_guest_agent/command/command_monitor.go
new file mode 100644
index 0000000..6e1ee86
--- /dev/null
+++ b/google_guest_agent/command/command_monitor.go
@@ -0,0 +1,228 @@
+//  Copyright 2023 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+/*
+ * This file contains the details of command's internal communication protocol
+ * listener. Most callers should not need to call anything in this file. The
+ * command handler and caller API is contained in command.go.
+ */
+
+package command
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"net"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
+	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
+)
+
+var cmdMonitor *Monitor = &Monitor{
+	handlersMu: new(sync.RWMutex),
+	handlers:   make(map[string]Handler),
+}
+
+// Init starts an internally managed command server. The agent configuration
+// will decide the server options. Returns a reference to the internally managed
+// command monitor which the caller can Close() when appropriate.
+func Init(ctx context.Context) {
+	if cmdMonitor.srv != nil {
+		return
+	}
+	pipe := cfg.Get().Unstable.CommandPipePath
+	if pipe == "" {
+		pipe = DefaultPipePath
+	}
+	to, err := time.ParseDuration(cfg.Get().Unstable.CommandRequestTimeout)
+	if err != nil {
+		logger.Errorf("commmand request timeout configuration is not a valid duration string, falling back to 10s timeout")
+		to = time.Duration(10) * time.Second
+	}
+	var pipemode int64 = 0770
+	pipemode, err = strconv.ParseInt(cfg.Get().Unstable.CommandPipeMode, 8, 32)
+	if err != nil {
+		logger.Errorf("could not parse command_pipe_mode as octal integer: %v falling back to mode 0770", err)
+	}
+	cmdMonitor.srv = &Server{
+		pipe:      pipe,
+		pipeMode:  int(pipemode),
+		pipeGroup: cfg.Get().Unstable.CommandPipeGroup,
+		timeout:   to,
+		monitor:   cmdMonitor,
+	}
+	err = cmdMonitor.srv.start(ctx)
+	if err != nil {
+		logger.Errorf("failed to start command server: %s", err)
+	}
+}
+
+// Close will close the internally managed command server, if it was initialized.
+func Close() error {
+	if cmdMonitor.srv != nil {
+		return cmdMonitor.srv.Close()
+	}
+	return nil
+}
+
+// Monitor is the structure which handles command registration and deregistration.
+type Monitor struct {
+	srv        *Server
+	handlersMu *sync.RWMutex
+	handlers   map[string]Handler
+}
+
+// Close stops the server from listening to commands.
+func (m *Monitor) Close() error { return m.srv.Close() }
+
+// Start begins listening for commands.
+func (m *Monitor) Start(ctx context.Context) error { return m.srv.start(ctx) }
+
+// Server is the server structure which will listen for command requests and
+// route them to handlers. Most callers should not interact with this directly.
+type Server struct {
+	pipe      string
+	pipeMode  int
+	pipeGroup string
+	timeout   time.Duration
+	srv       net.Listener
+	monitor   *Monitor
+}
+
+// Close signals the server to stop listening for commands and stop waiting to
+// listen.
+func (c *Server) Close() error {
+	if c.srv != nil {
+		return c.srv.Close()
+	}
+	return nil
+}
+
+func (c *Server) start(ctx context.Context) error {
+	if c.srv != nil {
+		return errors.New("server already listening")
+	}
+	srv, err := listen(ctx, c.pipe, c.pipeMode, c.pipeGroup)
+	if err != nil {
+		return err
+	}
+	go func() {
+		defer srv.Close()
+		for {
+			if ctx.Err() != nil {
+				return
+			}
+			conn, err := srv.Accept()
+			if err != nil {
+				if err == net.ErrClosed {
+					break
+				}
+				logger.Infof("error on connection to pipe %s: %v", c.pipe, err)
+				continue
+			}
+			go func(conn net.Conn) {
+				defer conn.Close()
+				// Go has lots of helpers to do this for us but none of them return the byte
+				// slice afterwards, and we need it for the handler
+				var b []byte
+				r := bufio.NewReader(conn)
+				var depth int
+				deadline := time.Now().Add(c.timeout)
+				e := conn.SetReadDeadline(deadline)
+				if e != nil {
+					logger.Infof("could not set read deadline on command request: %v", e)
+					return
+				}
+				for {
+					if time.Now().After(deadline) {
+						if b, err := json.Marshal(TimeoutError); err != nil {
+							conn.Write(internalError)
+						} else {
+							conn.Write(b)
+						}
+						return
+					}
+					rune, _, err := r.ReadRune()
+					if err != nil {
+						logger.Debugf("connection read error: %v", err)
+						if errors.Is(err, os.ErrDeadlineExceeded) {
+							if b, err := json.Marshal(TimeoutError); err != nil {
+								conn.Write(internalError)
+							} else {
+								conn.Write(b)
+							}
+						} else {
+							if b, err := json.Marshal(ConnError); err != nil {
+								conn.Write(internalError)
+							} else {
+								conn.Write(b)
+							}
+						}
+						return
+					}
+					b = append(b, byte(rune))
+					switch rune {
+					case '{':
+						depth++
+					case '}':
+						depth--
+					}
+					// Must check here because the first pass always depth = 0
+					if depth == 0 {
+						break
+					}
+				}
+				var req Request
+				err := json.Unmarshal(b, &req)
+				if err != nil {
+					if b, err := json.Marshal(BadRequestError); err != nil {
+						conn.Write(internalError)
+					} else {
+						conn.Write(b)
+					}
+					return
+				}
+				c.monitor.handlersMu.RLock()
+				defer c.monitor.handlersMu.RUnlock()
+				handler, ok := c.monitor.handlers[req.Command]
+				if !ok {
+					if b, err := json.Marshal(CmdNotFoundError); err != nil {
+						conn.Write(internalError)
+					} else {
+						conn.Write(b)
+					}
+					return
+				}
+				resp, err := handler(b)
+				if err != nil {
+					re := Response{Status: HandlerError.Status, StatusMessage: err.Error()}
+					if b, err := json.Marshal(re); err != nil {
+						resp = internalError
+					} else {
+						resp = b
+					}
+				}
+				conn.Write(resp)
+			}(conn)
+		}
+	}()
+	c.srv = srv
+	return nil
+}
diff --git a/google_guest_agent/command/command_test.go b/google_guest_agent/command/command_test.go
new file mode 100644
index 0000000..96c2d9f
--- /dev/null
+++ b/google_guest_agent/command/command_test.go
@@ -0,0 +1,209 @@
+//  Copyright 2023 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package command
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/rand"
+	"os/user"
+	"path"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
+)
+
+func cmdServerForTest(t *testing.T, pipeMode int, pipeGroup string, timeout time.Duration) *Server {
+	cs := &Server{
+		pipe:      getTestPipePath(t),
+		pipeMode:  pipeMode,
+		pipeGroup: pipeGroup,
+		timeout:   timeout,
+		monitor: &Monitor{
+			handlersMu: new(sync.RWMutex),
+			handlers:   make(map[string]Handler),
+		},
+	}
+	cs.monitor.srv = cs
+	err := cs.start(testctx(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		err := cs.Close()
+		if err != nil {
+			t.Errorf("error closing command server: %v", err)
+		}
+	})
+	return cs
+}
+
+func getTestPipePath(t *testing.T) string {
+	if runtime.GOOS == "windows" {
+		return `\\.\pipe\google-guest-agent-commands-test-` + t.Name()
+	}
+	return path.Join(t.TempDir(), "run", "pipe")
+}
+
+func testctx(t *testing.T) context.Context {
+	d, ok := t.Deadline()
+	if !ok {
+		ctx, cancel := context.WithCancel(context.Background())
+		t.Cleanup(cancel)
+		return ctx
+	}
+	ctx, cancel := context.WithDeadline(context.Background(), d)
+	t.Cleanup(cancel)
+	return ctx
+}
+
+type testRequest struct {
+	Command       string
+	ArbitraryData int
+}
+
+func TestInit(t *testing.T) {
+	cfg.Load(nil)
+	cfg.Get().Unstable.CommandPipePath = getTestPipePath(t)
+	if cmdMonitor.srv != nil {
+		t.Fatal("internal command server already exists")
+	}
+	Init(testctx(t))
+	if cmdMonitor.srv == nil {
+		t.Errorf("could not start internally managed command server")
+	}
+	if err := Close(); err != nil {
+		t.Errorf("could not close managed command server: %s", err)
+	}
+}
+
+func TestListen(t *testing.T) {
+	cu, err := user.Current()
+	if err != nil {
+		t.Fatalf("could not get current user: %v", err)
+	}
+	ug, err := cu.GroupIds()
+	if err != nil {
+		t.Fatalf("could not get user groups for %s: %v", cu.Name, err)
+	}
+	resp := []byte(`{"Status":0,"StatusMessage":"OK"}`)
+	errresp := []byte(`{"Status":1,"StatusMessage":"ERR"}`)
+	req := []byte(`{"ArbitraryData":1234,"Command":"TestListen"}`)
+	h := func(b []byte) ([]byte, error) {
+		var r testRequest
+		err := json.Unmarshal(b, &r)
+		if err != nil || r.ArbitraryData != 1234 {
+			return errresp, nil
+		}
+		return resp, nil
+	}
+
+	testcases := []struct {
+		name     string
+		filemode int
+		group    string
+	}{
+		{
+			name:     "world read/writeable",
+			filemode: 0777,
+			group:    "-1",
+		},
+		{
+			name:     "group read/writeable",
+			filemode: 0770,
+			group:    "-1",
+		},
+		{
+			name:     "user read/writeable",
+			filemode: 0700,
+			group:    "-1",
+		},
+		{
+			name:     "additional user group as group owner",
+			filemode: 0770,
+			group:    ug[rand.Intn(len(ug))],
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			cs := cmdServerForTest(t, tc.filemode, tc.group, time.Second)
+			err := cs.monitor.RegisterHandler("TestListen", h)
+			if err != nil {
+				t.Errorf("could not register handler: %v", err)
+			}
+			d := SendCmdPipe(testctx(t), cs.pipe, req)
+			var r Response
+			err = json.Unmarshal(d, &r)
+			if err != nil {
+				t.Error(err)
+			}
+			if r.Status != 0 || r.StatusMessage != "OK" {
+				t.Errorf("unexpected status from test-cmd, want 0, \"OK\" but got %d, %q", r.Status, r.StatusMessage)
+			}
+		})
+	}
+}
+
+func TestHandlerFailure(t *testing.T) {
+	req := []byte(`{"Command":"TestHandlerFailure"}`)
+	h := func(b []byte) ([]byte, error) {
+		return nil, fmt.Errorf("always fail")
+	}
+
+	cs := cmdServerForTest(t, 0777, "-1", time.Second)
+	cs.monitor.RegisterHandler("TestHandlerFailure", h)
+	d := SendCmdPipe(testctx(t), cs.pipe, req)
+	var r Response
+	err := json.Unmarshal(d, &r)
+	if err != nil {
+		t.Error(err)
+	}
+	if r.Status != HandlerError.Status || r.StatusMessage != "always fail" {
+		t.Errorf("unexpected status from TestHandlerFailure, want %d, \"always fail\" but got %d, %q", HandlerError.Status, r.Status, r.StatusMessage)
+	}
+}
+
+func TestListenTimeout(t *testing.T) {
+	expect, err := json.Marshal(TimeoutError)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if runtime.GOOS == "windows" {
+		// winio library does not surface timeouts from the underlying net.Conn as
+		// timeouts, but as generic errors. Timeouts still work they just can't be
+		// detected as timeouts, so they are generic connErrors here.
+		expect, err = json.Marshal(ConnError)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	cs := cmdServerForTest(t, 0770, "-1", time.Millisecond)
+	conn, err := dialPipe(testctx(t), cs.pipe)
+	if err != nil {
+		t.Errorf("could not connect to command server: %v", err)
+	}
+	data, err := io.ReadAll(conn)
+	if err != nil {
+		t.Errorf("error reading response from command server: %v", err)
+	}
+	if string(data) != string(expect) {
+		t.Errorf("unexpected response from timed out connection, got %s but want %s", data, expect)
+	}
+}
diff --git a/google_guest_agent/command/command_windows.go b/google_guest_agent/command/command_windows.go
new file mode 100644
index 0000000..0ac7213
--- /dev/null
+++ b/google_guest_agent/command/command_windows.go
@@ -0,0 +1,104 @@
+//  Copyright 2023 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package command
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os/user"
+
+	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
+	"github.com/Microsoft/go-winio"
+)
+
+const (
+	// DefaultPipePath is the default named pipe path for windows.
+	DefaultPipePath = `\\.\pipe\google-guest-agent-commands`
+	nullSID         = "S-1-0-0"
+	worldSID        = "S-1-1-0"
+	creatorOwnerSID = "S-1-3-0"
+	creatorGroupSID = "S-1-3-1"
+)
+
+func genSecurityDescriptor(filemode int, grp string) string {
+	// This function translates the intention of a unix file mode and owner group into an appropriate SDDL security descriptor for a windows named pipe.
+	owner := creatorOwnerSID
+	group := creatorGroupSID
+
+	wPerm := filemode % 010
+	filemode /= 010
+	gPerm := filemode % 010
+	filemode /= 010
+	uPerm := filemode % 010
+
+	// Having only read or only write access to a bidirectional pipe is pointless so we treat access for user/group as yes or no based on whether the permission grants RW access
+	if uPerm < 06 {
+		owner = nullSID
+	}
+	if gPerm < 06 {
+		group = nullSID
+	}
+	// If permissions grant world RW, make world the owner
+	if wPerm > 05 {
+		owner = worldSID
+		group = worldSID
+	}
+
+	// Group is handled as supplemental DACL, but ignore it if user specified no group rw permission
+	var dacl string
+	if gPerm > 05 {
+		g, err := user.LookupGroup(grp)
+		if err != nil {
+			logger.Errorf("Could not lookup group %s SID, this group will not be included in the command server security descriptor: %v", grp, err)
+		} else {
+			// Allow access;Protected DACL;Allow all general access;Empty object guid;Empty inherit object guid;group sid from lookup
+			dacl = fmt.Sprintf("D:(A;P;GA;;;%s)", g.Gid)
+		}
+	}
+
+	sddl := "O:%sG:%s%s"
+	return fmt.Sprintf(sddl, owner, group, dacl)
+}
+
+func listen(ctx context.Context, path string, filemode int, group string) (net.Listener, error) {
+	// Winio library does not provide any method to listen on context. Failing to
+	// specify a pipeconfig (or using the zero value) results in flaky ACCESS_DENIED
+	// errors when re-opening the same pipe (~1/10).
+	// https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea#remarks
+	// Even with a pipeconfig, this flakes ~1/200 runs, hence the retry until the
+	// context is expired or listen is successful.
+	var l net.Listener
+	var lastError error
+	for {
+		if ctx.Err() != nil {
+			return nil, fmt.Errorf("context expired: %v before successful listen (last error: %v)", ctx.Err(), lastError)
+		}
+		config := &winio.PipeConfig{
+			MessageMode:        false,
+			InputBufferSize:    1024,
+			OutputBufferSize:   1024,
+			SecurityDescriptor: genSecurityDescriptor(filemode, group),
+		}
+		l, lastError = winio.ListenPipe(path, config)
+		if lastError == nil {
+			return l, lastError
+		}
+	}
+}
+
+func dialPipe(ctx context.Context, pipe string) (net.Conn, error) {
+	return winio.DialPipeContext(ctx, pipe)
+}
diff --git a/google_guest_agent/command/command_windows_test.go b/google_guest_agent/command/command_windows_test.go
new file mode 100644
index 0000000..b4789e2
--- /dev/null
+++ b/google_guest_agent/command/command_windows_test.go
@@ -0,0 +1,73 @@
+//go:build windows
+
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package command
+
+import (
+	"os/user"
+	"testing"
+)
+
+func TestGenSecurityDescriptor(t *testing.T) {
+	guest, err := user.LookupGroup("Guests")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testcases := []struct {
+		name     string
+		filemode int
+		group    string
+		output   string
+	}{
+		{
+			name:     "world writeable",
+			filemode: 0777,
+			group:    nullSID,
+			output:   "O:" + worldSID + "G:" + worldSID,
+		},
+		{
+			name:     "user+group writable",
+			filemode: 0770,
+			group:    "",
+			output:   "O:" + creatorOwnerSID + "G:" + creatorGroupSID,
+		},
+		{
+			name:     "user writable",
+			filemode: 0700,
+			group:    nullSID,
+			output:   "O:" + creatorOwnerSID + "G:" + nullSID,
+		},
+		{
+			name:     "no write permissions",
+			filemode: 000,
+			group:    nullSID,
+			output:   "O:" + nullSID + "G:" + nullSID,
+		},
+		{
+			name:     "custom named group",
+			filemode: 0770,
+			group:    "Guests",
+			output:   "O:" + creatorOwnerSID + "G:" + creatorGroupSID + "D:(A;P;GA;;;" + guest.Gid + ")",
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			sd := genSecurityDescriptor(tc.filemode, tc.group)
+			if sd != tc.output {
+				t.Errorf("unexpected output from genSecurityDescriptor(%d, %s), got %s want %s", tc.filemode, tc.group, sd, tc.output)
+			}
+		})
+	}
+}
diff --git a/google_guest_agent/diagnostics.go b/google_guest_agent/diagnostics.go
index b2d239b..62cde39 100644
--- a/google_guest_agent/diagnostics.go
+++ b/google_guest_agent/diagnostics.go
@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"reflect"
 	"runtime"
+	"slices"
 	"sync/atomic"
 
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
@@ -94,7 +95,7 @@ func (d *diagnosticsMgr) Set(ctx context.Context) error {
 	}
 
 	strEntry := newMetadata.Instance.Attributes.Diagnostics
-	if utils.ContainsString(strEntry, diagnosticsEntries) {
+	if slices.Contains(diagnosticsEntries, strEntry) {
 		return nil
 	}
 	diagnosticsEntries = append(diagnosticsEntries, strEntry)
diff --git a/google_guest_agent/events/events.go b/google_guest_agent/events/events.go
index 2bf57e3..24d10dd 100644
--- a/google_guest_agent/events/events.go
+++ b/google_guest_agent/events/events.go
@@ -21,13 +21,14 @@ import (
 	"sync"
 
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/events/metadata"
-	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/events/sshtrustedca"
 	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
 )
 
 var (
-	// availableWatchers mapps all kown available event watchers.
-	availableWatchers = make(map[string]Watcher)
+	defaultWatchers = []Watcher{
+		metadata.New(),
+	}
+	instance *Manager
 )
 
 // Watcher defines the interface between the events manager and the actual
@@ -48,14 +49,65 @@ type Watcher interface {
 // Manager defines the interface between events management layer and the
 // core guest agent implementation.
 type Manager struct {
-	watchers    map[string]Watcher
+	// watcherEvents maps the registered watchers and their events.
+	watcherEvents []*WatcherEventType
+
+	// watchersMap is a convenient manager's mapping of registered watcher instances.
+	watchersMap map[string]bool
+
+	// watchersMutex protects the watchers map.
+	watchersMutex sync.Mutex
+
+	// removingWatcherEvents is a map of watchers being removed.
+	removingWatcherEvents map[string]bool
+
+	// running is a flag indicating if the Run() was previously called.
+	running bool
+
+	// runningMutex protects the running flag.
+	runningMutex sync.RWMutex
+
+	// subscribers maps the subscribed callbacks.
 	subscribers map[string][]*eventSubscriber
+
+	// subscribersMutex protects subscribers member/map of the manager object.
+	subscribersMutex sync.Mutex
+
+	// queue queue struct manages the running watchers, when it gets to len()
+	// down to zero means all watchers are done and we can signal the other
+	// control go routines to leave(given we don't have any more job left to
+	// process).
+	queue *watcherQueue
 }
 
-// Config offers a mechanism for the consumers to configure the Manager behavior.
-type Config struct {
-	// Watchers lists the enabled watchers, of not provided all available watchers will be enabled.
-	Watchers []string
+// watcherQueue wraps the watchers <-> callbacks communication as well as the
+// communication/coordination of the multiple control go routine i.e. the one
+// responsible to calling callbacks after a event is produced by the watcher etc.
+type watcherQueue struct {
+	// queueMutex protects the access to watchersMap
+	queueMutex sync.RWMutex
+
+	// watchersMap maps the currently running watchers.
+	watchersMap map[string]bool
+
+	// finishContextHandler is a channel used to communicate with the context handling
+	// go routine that it should finish/end its job (usually after all watchers are done).
+	finishContextHandler chan bool
+
+	// finishCallbackHandler is a channel used to communicate with the callback handling
+	// go routine that it should finish/end its job (usually after all watchers are done).
+	finishCallbackHandler chan bool
+
+	// watcherDone is a channel used to communicate that a given watcher is finished/done.
+	watcherDone chan string
+
+	// dataBus is the channel used to communicate between watchers (event producer) and the
+	// callback handler (event consumer managing go routine).
+	dataBus chan eventBusData
+
+	// leaving is a flag that indicates no more job should be processed as we are done
+	// with all watchers and callbacks.
+	leaving bool
 }
 
 // EventData wraps the data communicated from a Watcher to a Subscriber.
@@ -72,6 +124,20 @@ type WatcherEventType struct {
 	watcher Watcher
 	// evType idenfities the event type this object refences to.
 	evType string
+	// removed is a channel used to communicate with the running watcher go routine
+	// that it shouldn't renew even if the watcher requested a renew (in response of a
+	// RemoveWatcher() call.
+	removed chan bool
+}
+
+type eventSubscriber struct {
+	data interface{}
+	cb   *EventCb
+}
+
+type eventBusData struct {
+	evType string
+	data   *EventData
 }
 
 // EventCb defines the callback interface between watchers and subscribers. The arguments are:
@@ -84,224 +150,328 @@ type WatcherEventType struct {
 // to be unregistered/unsubscribed.
 type EventCb func(ctx context.Context, evType string, data interface{}, evData *EventData) bool
 
-type eventSubscriber struct {
-	data interface{}
-	cb   EventCb
+// length returns how many watchers are currently running.
+func (ep *watcherQueue) length() int {
+	ep.queueMutex.RLock()
+	defer ep.queueMutex.RUnlock()
+	return len(ep.watchersMap)
 }
 
-type eventBusData struct {
-	evType string
-	data   *EventData
+// add adds a new watcher to the queue.
+func (ep *watcherQueue) add(evType string) {
+	ep.queueMutex.Lock()
+	defer ep.queueMutex.Unlock()
+	ep.watchersMap[evType] = true
 }
 
-func init() {
-	err := initWatchers([]Watcher{
-		metadata.New(),
-		sshtrustedca.New(sshtrustedca.DefaultPipePath),
-	})
-	if err != nil {
-		logger.Errorf("Failed to initialize watchers: %+v", err)
-	}
+// del removes a watcher from the queue.
+func (ep *watcherQueue) del(evType string) int {
+	ep.queueMutex.Lock()
+	defer ep.queueMutex.Unlock()
+	delete(ep.watchersMap, evType)
+	return len(ep.watchersMap)
 }
 
-// init initializes the known available event watchers.
-func initWatchers(watchers []Watcher) error {
-	for _, curr := range watchers {
-		// Error if we are accidentaly not properly setting the id.
-		if curr.ID() == "" {
-			return fmt.Errorf("invalid event watcher id, skipping")
+// AddDefaultWatchers add the default watchers:
+//   - metadata
+func (mngr *Manager) AddDefaultWatchers(ctx context.Context) error {
+	for _, curr := range defaultWatchers {
+		if err := mngr.AddWatcher(ctx, curr); err != nil {
+			return err
 		}
-		availableWatchers[curr.ID()] = curr
 	}
 	return nil
 }
 
-// New allocates and initializes a events Manager based on provided cfg.
-func New(cfg *Config) (*Manager, error) {
-	res := &Manager{
-		watchers:    availableWatchers,
-		subscribers: make(map[string][]*eventSubscriber),
+// newManager allocates and initializes a events Manager.
+func newManager() *Manager {
+	return &Manager{
+		watchersMap:           make(map[string]bool),
+		removingWatcherEvents: make(map[string]bool),
+		subscribers:           make(map[string][]*eventSubscriber),
+		queue: &watcherQueue{
+			watchersMap:           make(map[string]bool),
+			dataBus:               make(chan eventBusData),
+			finishCallbackHandler: make(chan bool),
+			finishContextHandler:  make(chan bool),
+			watcherDone:           make(chan string),
+		},
 	}
+}
 
-	// Align manager's config based on consumers provided watchers If it is
-	// passing in wanted/expected watchers, otherwise use all available ones.
-	if cfg != nil && len(cfg.Watchers) > 0 {
-		res.watchers = make(map[string]Watcher)
-		for _, curr := range cfg.Watchers {
-			// Report back if we don't know the provided watcher id.
-			if _, found := availableWatchers[curr]; !found {
-				return nil, fmt.Errorf("invalid/unknown watcher id: %s", curr)
-			}
-			res.watchers[curr] = availableWatchers[curr]
-		}
-	}
+func init() {
+	instance = newManager()
+}
 
-	return res, nil
+// Get allocates a new manager if one doesn't exists or returns the one previously allocated.
+func Get() *Manager {
+	if instance == nil {
+		panic("The event's manager instance should had being initialized.")
+	}
+	return instance
 }
 
 // Subscribe registers an event consumer/subscriber callback to a given event type, data
 // is a context pointer provided by the caller to be passed down when calling cb when
 // a new event happens.
 func (mngr *Manager) Subscribe(evType string, data interface{}, cb EventCb) {
+	mngr.subscribersMutex.Lock()
+	defer mngr.subscribersMutex.Unlock()
 	mngr.subscribers[evType] = append(mngr.subscribers[evType],
 		&eventSubscriber{
 			data: data,
-			cb:   cb,
+			cb:   &cb,
 		},
 	)
 }
 
-func (mngr *Manager) eventTypes() []*WatcherEventType {
-	var res []*WatcherEventType
-	for _, watcher := range mngr.watchers {
-		for _, evType := range watcher.Events() {
-			res = append(res, &WatcherEventType{watcher, evType})
+func (mngr *Manager) unsubscribe(evType string, cb *EventCb) {
+	var keepMe []*eventSubscriber
+	for _, curr := range mngr.subscribers[evType] {
+		if curr.cb != cb {
+			keepMe = append(keepMe, curr)
 		}
 	}
-	return res
+
+	mngr.subscribers[evType] = keepMe
+
+	if len(keepMe) == 0 {
+		logger.Debugf("No more subscribers left for evType: %s", evType)
+		delete(mngr.subscribers, evType)
+	}
 }
 
-type watcherQueue struct {
-	mutex       sync.Mutex
-	watchersMap map[string]bool
+// Unsubscribe removes the subscription of a given callback for a given event type.
+func (mngr *Manager) Unsubscribe(evType string, cb EventCb) {
+	mngr.subscribersMutex.Lock()
+	defer mngr.subscribersMutex.Unlock()
+	mngr.unsubscribe(evType, &cb)
 }
 
-func (ep *watcherQueue) add(evType string) {
-	ep.mutex.Lock()
-	defer ep.mutex.Unlock()
-	ep.watchersMap[evType] = true
+// RemoveWatcher removes a watcher from the event manager. Each running watcher has its own
+// context (derived from the one provided in the AddWatcher() call) and will have it canceled
+// after calling this method.
+func (mngr *Manager) RemoveWatcher(ctx context.Context, watcher Watcher) error {
+	mngr.watchersMutex.Lock()
+	defer mngr.watchersMutex.Unlock()
+
+	id := watcher.ID()
+	logger.Debugf("Got a request to remove watcher: %s", id)
+	if _, found := mngr.watchersMap[id]; !found {
+		return fmt.Errorf("unknown Watcher(%s)", id)
+	}
+
+	for _, curr := range mngr.watcherEvents {
+		if _, found := mngr.removingWatcherEvents[curr.evType]; found {
+			logger.Debugf("Watcher(%s) is being removed, skipping removal request: %s", id, curr.evType)
+			continue
+		}
+
+		if curr.watcher.ID() == id {
+			mngr.removingWatcherEvents[curr.evType] = true
+			logger.Debugf("Removing watcher: %s, event type: %s", id, curr.evType)
+			curr.removed <- true
+		}
+	}
+
+	return nil
 }
 
-func (ep *watcherQueue) del(evType string) int {
-	ep.mutex.Lock()
-	defer ep.mutex.Unlock()
-	delete(ep.watchersMap, evType)
-	return len(ep.watchersMap)
+// AddWatcher adds/enables a new watcher. The watcher will be fired up right away if the
+// event manager is already running, otherwise it's scheduled to run when Run() is called.
+func (mngr *Manager) AddWatcher(ctx context.Context, watcher Watcher) error {
+	mngr.watchersMutex.Lock()
+	defer mngr.watchersMutex.Unlock()
+	id := watcher.ID()
+	if _, found := mngr.watchersMap[id]; found {
+		return fmt.Errorf("watcher(%s) was previously added", id)
+	}
+
+	// Add the watchers and its events to internal mappings.
+	evTypes := make(map[string]*WatcherEventType)
+	mngr.watchersMap[id] = true
+
+	for _, curr := range watcher.Events() {
+		evType := &WatcherEventType{
+			watcher: watcher,
+			evType:  curr,
+			removed: make(chan bool),
+		}
+
+		evTypes[curr] = evType
+		mngr.watcherEvents = append(mngr.watcherEvents, evType)
+	}
+
+	mngr.runningMutex.RLock()
+	defer mngr.runningMutex.RUnlock()
+	// If we are not running don't bother "running" the watcher, Run() will do it later.
+	if !mngr.running {
+		return nil
+	}
+
+	// If we are already running the "run/launch" the watcher.
+	for _, curr := range watcher.Events() {
+		logger.Debugf("Adding watcher for event: %s", curr)
+		mngr.queue.add(curr)
+		go func(watcher Watcher, evType string, removed chan bool) {
+			mngr.runWatcher(ctx, watcher, evType, removed)
+		}(watcher, curr, evTypes[curr].removed)
+	}
+
+	return nil
+}
+
+func (mngr *Manager) runWatcher(ctx context.Context, watcher Watcher, evType string, removed chan bool) {
+	nCtx, cancel := context.WithCancel(ctx)
+	abort := false
+	id := watcher.ID()
+
+	go func() {
+		abort = <-removed
+		logger.Debugf("Got a request to abort watcher(%s) for event: %s", id, evType)
+		cancel()
+	}()
+
+	for renew := true; renew; {
+		var evData interface{}
+		var err error
+
+		renew, evData, err = watcher.Run(nCtx, evType)
+
+		logger.Debugf("Watcher(%s) returned event: %q, should renew?: %t", id, evType, renew)
+
+		if abort || mngr.queue.leaving {
+			logger.Debugf("Watcher(%s), either are aborting(%t) or leaving(%t), breaking renew cycle",
+				id, abort, mngr.queue.leaving)
+			break
+		}
+
+		mngr.queue.dataBus <- eventBusData{
+			evType: evType,
+			data: &EventData{
+				Data:  evData,
+				Error: err,
+			},
+		}
+	}
+
+	logger.Debugf("watcher finishing: %s", evType)
+	if !abort {
+		removed <- true
+	}
+
+	mngr.queue.watcherDone <- evType
 }
 
 // Run runs the event manager, it will block until all watchers have given up/failed.
-func (mngr *Manager) Run(ctx context.Context) {
+// The event manager is meant to be started right after the early initialization code
+// and live until the application ends, the event manager can not be restarted - the Run()
+// method will return an error if one tries to run it twice.
+func (mngr *Manager) Run(ctx context.Context) error {
 	var wg sync.WaitGroup
-	var leaving bool
-
-	syncBus := make(chan eventBusData)
-	defer close(syncBus)
 
-	cancelContext := make(chan bool)
-	cancelCallback := make(chan bool)
+	mngr.runningMutex.Lock()
+	if mngr.running {
+		mngr.runningMutex.Unlock()
+		return fmt.Errorf("tried calling event manager's Run() twice")
+	}
+	mngr.running = true
+	mngr.runningMutex.Unlock()
 
-	defer close(cancelContext)
-	defer close(cancelCallback)
+	queue := mngr.queue
 
 	// Manages the context's done signal, pass it down to the other go routines to
 	// finish its job and leave. Additionally, if the remaining go routines are leaving
-	// we get it handled via syncBus channel and drop this go routine as well.
+	// we get it handled via dataBus channel and drop this go routine as well.
 	wg.Add(1)
-	go func(done <-chan struct{}, cancelContext <-chan bool, cancelCallback chan<- bool) {
+	go func(done <-chan struct{}, finishContextHandler <-chan bool, finishCallbackHandler chan<- bool) {
 		defer wg.Done()
 
 		for {
 			select {
 			case <-done:
 				logger.Debugf("Got context's Done() signal, leaving.")
-				leaving = true
-				cancelCallback <- true
+				queue.leaving = true
+				finishCallbackHandler <- true
 				return
-			case <-cancelContext:
-				leaving = true
+			case <-finishContextHandler:
+				logger.Debugf("Got context handler finish signal, leaving.")
+				queue.leaving = true
 				return
 			}
 		}
-	}(ctx.Done(), cancelContext, cancelCallback)
+	}(ctx.Done(), queue.finishContextHandler, queue.finishCallbackHandler)
 
 	// Manages the event processing avoiding blocking the watcher's go routines.
-	// This will listen to syncBus and call the events handlers/callbacks.
+	// This will listen to dataBus and call the events handlers/callbacks.
 	wg.Add(1)
-	go func(bus <-chan eventBusData, cancelCallback <-chan bool) {
+	go func(bus <-chan eventBusData, finishCallbackHandler <-chan bool) {
 		defer wg.Done()
 
 		for {
 			select {
-			case <-cancelCallback:
+			case <-finishCallbackHandler:
 				return
 			case busData := <-bus:
-				subscribers, found := mngr.subscribers[busData.evType]
-				if !found || len(subscribers) == 0 {
+				subscribers := mngr.subscribers[busData.evType]
+				if subscribers == nil {
 					logger.Debugf("No subscriber found for event: %s, returning.", busData.evType)
 					continue
 				}
 
-				keepMe := make([]*eventSubscriber, 0)
-				for _, curr := range mngr.subscribers[busData.evType] {
+				deleteMe := make([]*eventSubscriber, 0)
+				for _, curr := range subscribers {
 					logger.Debugf("Running registered callback for event: %s", busData.evType)
-					renew := curr.cb(ctx, busData.evType, curr.data, busData.data)
-					if renew {
-						keepMe = append(keepMe, curr)
+					renew := (*curr.cb)(ctx, busData.evType, curr.data, busData.data)
+					if !renew {
+						deleteMe = append(deleteMe, curr)
 					}
 					logger.Debugf("Returning from event %q subscribed callback, should renew?: %t", busData.evType, renew)
 				}
 
-				mngr.subscribers[busData.evType] = keepMe
-
-				// No more subscribers for this event type, delete it from the subscribers map.
-				if len(keepMe) == 0 {
-					logger.Debugf("No more subscribers left for evType: %s", busData.evType)
-					delete(mngr.subscribers, busData.evType)
+				mngr.subscribersMutex.Lock()
+				for _, curr := range deleteMe {
+					mngr.unsubscribe(busData.evType, curr.cb)
 				}
+				leave := mngr.subscribers[busData.evType] == nil
+				mngr.subscribersMutex.Unlock()
 
 				// No more subscribers at all, we have nothing more left to do here.
-				if len(mngr.subscribers) == 0 {
+				if leave {
 					logger.Debugf("No subscribers left, leaving")
 					break
 				}
 			}
 		}
-	}(syncBus, cancelCallback)
-
-	// This control struct manages the registered watchers, when it gets to len()
-	// down to zero means all watchers are done and we can signal the other 2 control
-	// go routines to leave(given we don't have any more job left to process).
-	control := &watcherQueue{
-		watchersMap: make(map[string]bool),
-	}
+	}(queue.dataBus, queue.finishCallbackHandler)
 
 	// Creates a goroutine for each registered watcher's event and keep handling its
 	// execution until they give up/finishes their job by returning renew = false.
-	for _, curr := range mngr.eventTypes() {
-		control.add(curr.evType)
-		wg.Add(1)
-
-		go func(bus chan<- eventBusData, watcher Watcher, evType string, cancelContext chan<- bool, cancelCallback chan<- bool) {
-			var evData interface{}
-			var err error
-
-			defer wg.Done()
-
-			for renew := true; renew; {
-				renew, evData, err = watcher.Run(ctx, evType)
-
-				logger.Debugf("Watcher(%s) returned event: %q, should renew?: %t", watcher.ID(), evType, renew)
-
-				if leaving {
-					break
-				}
+	for _, curr := range mngr.watcherEvents {
+		queue.add(curr.evType)
+		go func(watcher Watcher, evType string, removed chan bool) {
+			mngr.runWatcher(ctx, watcher, evType, removed)
+		}(curr.watcher, curr.evType, curr.removed)
+	}
 
-				bus <- eventBusData{
-					evType: evType,
-					data: &EventData{
-						Data:  evData,
-						Error: err,
-					},
-				}
-			}
+	// Controls the completion of the watcher go routines, their removal from the queue
+	// and signals to context & callback control go routines about watchers completion.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
 
-			if !leaving && control.del(evType) == 0 {
+		for len := queue.length(); len > 0; {
+			doneStr := <-queue.watcherDone
+			len = queue.del(doneStr)
+			delete(mngr.removingWatcherEvents, doneStr)
+			if !queue.leaving && len == 0 {
 				logger.Debugf("All watchers are finished, signaling to leave.")
-				cancelContext <- true
-				cancelCallback <- true
+				queue.finishContextHandler <- true
+				queue.finishCallbackHandler <- true
 			}
-		}(syncBus, curr.watcher, curr.evType, cancelContext, cancelCallback)
-	}
+		}
+	}()
 
 	wg.Wait()
+	return nil
 }
diff --git a/google_guest_agent/events/events_test.go b/google_guest_agent/events/events_test.go
index 6fcb2c3..a77bfb9 100644
--- a/google_guest_agent/events/events_test.go
+++ b/google_guest_agent/events/events_test.go
@@ -17,48 +17,26 @@ package events
 import (
 	"context"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/events/metadata"
 )
 
-func TestConstructor(t *testing.T) {
-	tests := []struct {
-		config  *Config
-		success bool
-	}{
-		{config: nil, success: true},
-		{config: &Config{Watchers: []string{metadata.WatcherID}}, success: true},
-		{config: &Config{Watchers: []string{"foobar"}}, success: false},
-	}
+func TestAddWatcher(t *testing.T) {
+	eventManager := newManager()
+	metadataWatcher := metadata.New()
+	ctx := context.Background()
 
-	for i, tt := range tests {
-		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
-			_, err := New(tt.config)
-			if err != nil && tt.success {
-				t.Errorf("expected success, got error: %+v", err)
-			}
-		})
-	}
-}
-
-func TestInitWatcers(t *testing.T) {
-	tests := []struct {
-		watchers []Watcher
-		success  bool
-	}{
-		{watchers: []Watcher{metadata.New()}, success: true},
-		{watchers: []Watcher{&testWatcher{}}, success: false},
+	err := eventManager.AddWatcher(ctx, metadataWatcher)
+	if err != nil {
+		t.Errorf("expected success, got error: %+v", err)
 	}
 
-	for i, tt := range tests {
-		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
-			err := initWatchers(tt.watchers)
-			if err != nil && tt.success {
-				t.Errorf("expected success, got error: %+v", err)
-			}
-		})
+	err = eventManager.AddWatcher(ctx, metadataWatcher)
+	if err == nil {
+		t.Errorf("expected error, had success, event manager shouldn't add same watcher twice")
 	}
 }
 
@@ -91,20 +69,16 @@ func TestRun(t *testing.T) {
 	watcherID := "test-watcher"
 	maxCount := 10
 
-	err := initWatchers([]Watcher{
-		&testWatcher{
-			watcherID: watcherID,
-			maxCount:  maxCount,
-		},
-	})
+	ctx := context.Background()
+	eventManager := newManager()
 
-	if err != nil {
-		t.Fatalf("Failed to init/register watcher: %+v", err)
-	}
+	err := eventManager.AddWatcher(ctx, &testWatcher{
+		watcherID: watcherID,
+		maxCount:  maxCount,
+	})
 
-	eventManager, err := New(&Config{Watchers: []string{watcherID}})
 	if err != nil {
-		t.Fatalf("Failed to init event manager: %+v", err)
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
 	}
 
 	counter := 0
@@ -114,7 +88,9 @@ func TestRun(t *testing.T) {
 		return true
 	})
 
-	eventManager.Run(context.Background())
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+	}
 
 	if counter != maxCount {
 		t.Errorf("Failed to increment callback counter, expected: %d, got: %d", maxCount, counter)
@@ -126,20 +102,16 @@ func TestUnsubscribe(t *testing.T) {
 	maxCount := 10
 	unsubscribeAt := 2
 
-	err := initWatchers([]Watcher{
-		&testWatcher{
-			watcherID: watcherID,
-			maxCount:  maxCount,
-		},
-	})
+	ctx := context.Background()
+	eventManager := newManager()
 
-	if err != nil {
-		t.Fatalf("Failed to init/register watcher: %+v", err)
-	}
+	err := eventManager.AddWatcher(ctx, &testWatcher{
+		watcherID: watcherID,
+		maxCount:  maxCount,
+	})
 
-	eventManager, err := New(&Config{Watchers: []string{watcherID}})
 	if err != nil {
-		t.Fatalf("Failed to init event manager: %+v", err)
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
 	}
 
 	counter := 0
@@ -151,7 +123,9 @@ func TestUnsubscribe(t *testing.T) {
 		return true
 	})
 
-	eventManager.Run(context.Background())
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+	}
 
 	if counter != unsubscribeAt {
 		t.Errorf("Failed to unsubscribe callback, expected: %d, got: %d", unsubscribeAt, counter)
@@ -162,20 +136,16 @@ func TestCancelBeforeCallbacks(t *testing.T) {
 	watcherID := "test-watcher"
 	timeout := (1 * time.Second) / 100
 
-	err := initWatchers([]Watcher{
-		&testCancel{
-			watcherID: watcherID,
-			timeout:   timeout,
-		},
-	})
+	ctx, cancel := context.WithCancel(context.Background())
+	eventManager := newManager()
 
-	if err != nil {
-		t.Fatalf("Failed to init/register watcher: %+v", err)
-	}
+	err := eventManager.AddWatcher(ctx, &testCancel{
+		watcherID: watcherID,
+		timeout:   timeout,
+	})
 
-	eventManager, err := New(&Config{Watchers: []string{watcherID}})
 	if err != nil {
-		t.Fatalf("Failed to init event manager: %+v", err)
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
 	}
 
 	eventManager.Subscribe("test-watcher,test-event", nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
@@ -183,13 +153,14 @@ func TestCancelBeforeCallbacks(t *testing.T) {
 		return true
 	})
 
-	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
 		time.Sleep(timeout / 2)
 		cancel()
 	}()
 
-	eventManager.Run(ctx)
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+	}
 }
 
 type testCancel struct {
@@ -214,33 +185,30 @@ func TestCancelAfterCallbacks(t *testing.T) {
 	watcherID := "test-watcher"
 	timeout := (1 * time.Second) / 100
 
-	err := initWatchers([]Watcher{
-		&testCancel{
-			watcherID: watcherID,
-			timeout:   timeout,
-		},
-	})
+	ctx, cancel := context.WithCancel(context.Background())
+	eventManager := newManager()
 
-	if err != nil {
-		t.Fatalf("Failed to init/register watcher: %+v", err)
-	}
+	err := eventManager.AddWatcher(ctx, &testCancel{
+		watcherID: watcherID,
+		timeout:   timeout,
+	})
 
-	eventManager, err := New(&Config{Watchers: []string{watcherID}})
 	if err != nil {
-		t.Fatalf("Failed to init event manager: %+v", err)
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
 	}
 
 	eventManager.Subscribe("test-watcher,test-event", nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
 		return true
 	})
 
-	ctx, cancel := context.WithCancel(context.Background())
 	go func() {
 		time.Sleep(timeout * 10)
 		cancel()
 	}()
 
-	eventManager.Run(ctx)
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+	}
 }
 
 type testCancelWatcher struct {
@@ -285,20 +253,16 @@ func TestCancelCallbacksAndWatchers(t *testing.T) {
 		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
 			cancelSubscriberAfter := curr.cancelSubscriberAfter
 
-			err := initWatchers([]Watcher{
-				&testCancelWatcher{
-					watcherID: watcherID,
-					after:     curr.cancelWatcherAfter,
-				},
-			})
+			ctx := context.Background()
+			eventManager := newManager()
 
-			if err != nil {
-				t.Fatalf("Failed to init/register watcher: %+v", err)
-			}
+			err := eventManager.AddWatcher(ctx, &testCancelWatcher{
+				watcherID: watcherID,
+				after:     curr.cancelWatcherAfter,
+			})
 
-			eventManager, err := New(&Config{Watchers: []string{watcherID}})
 			if err != nil {
-				t.Fatalf("Failed to init event manager: %+v", err)
+				t.Fatalf("Failed to add watcher to event manager: %+v", err)
 			}
 
 			eventManager.Subscribe("test-watcher,test-event", nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
@@ -310,7 +274,9 @@ func TestCancelCallbacksAndWatchers(t *testing.T) {
 				return true
 			})
 
-			eventManager.Run(context.Background())
+			if err := eventManager.Run(ctx); err != nil {
+				t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+			}
 		})
 	}
 }
@@ -320,20 +286,16 @@ func TestMultipleEvents(t *testing.T) {
 	firstEvent := "multiple-events,first-event"
 	secondEvent := "multiple-events,second-event"
 
-	err := initWatchers([]Watcher{
-		&testMultipleEvents{
-			watcherID: watcherID,
-			eventIDS:  []string{firstEvent, secondEvent},
-		},
-	})
+	ctx := context.Background()
+	eventManager := newManager()
 
-	if err != nil {
-		t.Fatalf("Failed to init/register watcher: %+v", err)
-	}
+	err := eventManager.AddWatcher(ctx, &testMultipleEvents{
+		watcherID: watcherID,
+		eventIDS:  []string{firstEvent, secondEvent},
+	})
 
-	eventManager, err := New(&Config{Watchers: []string{watcherID}})
 	if err != nil {
-		t.Fatalf("Failed to init event manager: %+v", err)
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
 	}
 
 	var hitFirstEvent bool
@@ -348,7 +310,9 @@ func TestMultipleEvents(t *testing.T) {
 		return false
 	})
 
-	eventManager.Run(context.Background())
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+	}
 
 	if !hitFirstEvent || !hitSecondEvent {
 		t.Errorf("Failed to call back events, first event hit? (%t), second event hit? (%t)", hitFirstEvent, hitSecondEvent)
@@ -371,3 +335,304 @@ func (tt *testMultipleEvents) Events() []string {
 func (tt *testMultipleEvents) Run(ctx context.Context, evType string) (bool, interface{}, error) {
 	return false, nil, nil
 }
+
+func TestAddWatcherAfterRun(t *testing.T) {
+	firstWatcher := &genericWatcher{
+		watcherID:   "first-watcher",
+		shouldRenew: true,
+	}
+
+	secondWatcher := &genericWatcher{
+		watcherID: "second-watcher",
+	}
+
+	ctx := context.Background()
+	eventManager := newManager()
+
+	err := eventManager.AddWatcher(ctx, firstWatcher)
+
+	if err != nil {
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
+	}
+
+	eventManager.Subscribe(firstWatcher.eventID(), nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
+		if err := eventManager.AddWatcher(ctx, secondWatcher); err != nil {
+			t.Errorf("Failed to add a second watcher: %+v, expected success", err)
+		}
+		firstWatcher.shouldRenew = false
+		return false
+	})
+
+	var hitSecondEvent bool
+	eventManager.Subscribe(secondWatcher.eventID(), nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
+		hitSecondEvent = true
+		return false
+	})
+
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed to run event managed, expected success, got error: %+v", err)
+	}
+
+	if !hitSecondEvent {
+		t.Errorf("Failed registering second watcher, expected hitSecondEvent: false, got: %t", hitSecondEvent)
+	}
+}
+
+type genericWatcher struct {
+	watcherID   string
+	shouldRenew bool
+	wait        time.Duration
+}
+
+func (gw *genericWatcher) eventID() string {
+	return gw.watcherID + ",test-event"
+}
+
+func (gw *genericWatcher) ID() string {
+	return gw.watcherID
+}
+
+func (gw *genericWatcher) Events() []string {
+	return []string{gw.eventID()}
+}
+
+func (gw *genericWatcher) Run(ctx context.Context, evType string) (bool, interface{}, error) {
+	if gw.wait > 0 {
+		time.Sleep(gw.wait)
+	}
+	return gw.shouldRenew, nil, nil
+}
+
+func TestAddDefaultWatchers(t *testing.T) {
+	firstWatcher := &genericWatcher{
+		watcherID:   "first-watcher",
+		shouldRenew: false,
+	}
+
+	defaultWatchers = []Watcher{
+		firstWatcher,
+	}
+
+	ctx := context.Background()
+	eventManager := newManager()
+
+	err := eventManager.AddDefaultWatchers(ctx)
+
+	if err != nil {
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
+	}
+
+	if len(eventManager.watchersMap) == 0 {
+		t.Fatalf("Failed to add default watchers, expected: %d, got: %d", len(defaultWatchers),
+			len(eventManager.watchersMap))
+	}
+
+	if len(eventManager.watcherEvents) == 0 {
+		t.Fatalf("Failed to add default watchers, expected: %d, got: %d", len(defaultWatchers),
+			len(eventManager.watcherEvents))
+	}
+}
+
+func TestCallingRunTwice(t *testing.T) {
+	firstWatcher := &genericWatcher{
+		watcherID:   "first-watcher",
+		shouldRenew: false,
+	}
+
+	defaultWatchers = []Watcher{
+		firstWatcher,
+	}
+
+	timeout := (1 * time.Second) / 100
+	ctx, cancel := context.WithCancel(context.Background())
+	eventManager := newManager()
+
+	err := eventManager.AddDefaultWatchers(ctx)
+
+	if err != nil {
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		time.Sleep(timeout)
+		cancel()
+	}()
+
+	errors := []error{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := eventManager.Run(ctx); err != nil {
+			errors = append(errors, err)
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := eventManager.Run(ctx); err != nil {
+			errors = append(errors, err)
+		}
+	}()
+
+	wg.Wait()
+
+	if len(errors) == 0 {
+		t.Errorf("Executing Run() twice should fail, we got not failure")
+	}
+
+	if len(errors) > 1 {
+		t.Errorf("Executing Run() twice should produce a single error, got: %+v", errors)
+	}
+}
+
+type testRemoveWatcher struct {
+	watcherID string
+	timeout   time.Duration
+}
+
+func (tc *testRemoveWatcher) ID() string {
+	return tc.watcherID
+}
+
+func (tc *testRemoveWatcher) Events() []string {
+	return []string{tc.watcherID + ",test-event"}
+}
+
+func (tc *testRemoveWatcher) Run(ctx context.Context, evType string) (bool, interface{}, error) {
+	select {
+	case <-ctx.Done():
+		return false, nil, nil
+	case <-time.After(tc.timeout):
+		return true, nil, nil
+	}
+}
+
+func TestRemoveWatcherBeforeCallbacks(t *testing.T) {
+	watcherID := "test-watcher"
+	timeout := (1 * time.Second) / 100
+
+	ctx := context.Background()
+	eventManager := newManager()
+
+	watcher := &testRemoveWatcher{
+		watcherID: watcherID,
+		timeout:   timeout,
+	}
+
+	err := eventManager.AddWatcher(ctx, watcher)
+
+	if err != nil {
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
+	}
+
+	eventManager.Subscribe("test-watcher,test-event", nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
+		t.Errorf("Expected to have canceled before calling callback")
+		return false
+	})
+
+	go func() {
+		time.Sleep(timeout / 2)
+		if err := eventManager.RemoveWatcher(ctx, watcher); err != nil {
+			t.Errorf("Failed to remove watcher: %+v", err)
+		}
+	}()
+
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed running event manager, expected success, got error: %+v", err)
+	}
+}
+
+func TestRemoveWatcherFromCallback(t *testing.T) {
+	watcher := &genericWatcher{
+		watcherID:   "first-watcher",
+		shouldRenew: true,
+	}
+
+	ctx := context.Background()
+	eventManager := newManager()
+
+	err := eventManager.AddWatcher(ctx, watcher)
+
+	if err != nil {
+		t.Fatalf("Failed to add watcher to event manager: %+v", err)
+	}
+
+	eventManager.Subscribe(watcher.eventID(), nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
+		if err := eventManager.RemoveWatcher(ctx, watcher); err != nil {
+			t.Fatalf("Failed to remove watcher, it should have succeeded: %+v", err)
+		}
+		return true
+	})
+
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed running event manager, expected success, got error: %+v", err)
+	}
+}
+
+func TestCrossWatcherRemovalFromCallback(t *testing.T) {
+	firstWatcher := &genericWatcher{
+		watcherID:   "first-watcher",
+		shouldRenew: true,
+	}
+
+	secondWatcher := &genericWatcher{
+		watcherID:   "second-watcher",
+		shouldRenew: true,
+	}
+
+	thirdWatcher := &genericWatcher{
+		watcherID:   "third-watcher",
+		shouldRenew: true,
+		wait:        (1 * time.Second) / 3,
+	}
+
+	ctx := context.Background()
+	eventManager := newManager()
+
+	watchers := []Watcher{
+		firstWatcher,
+		secondWatcher,
+		thirdWatcher,
+	}
+
+	for _, curr := range watchers {
+		err := eventManager.AddWatcher(ctx, curr)
+
+		if err != nil {
+			t.Fatalf("Failed to add watcher to event manager: %+v", err)
+		}
+	}
+
+	removed := false
+	eventManager.Subscribe(thirdWatcher.eventID(), nil, func(ctx context.Context, evType string, data interface{}, evData *EventData) bool {
+		if !removed {
+			if err := eventManager.RemoveWatcher(ctx, firstWatcher); err != nil {
+				t.Errorf("Failed to remove firstWatcher, it should have succeeded: %+v", err)
+			}
+			if err := eventManager.RemoveWatcher(ctx, secondWatcher); err != nil {
+				t.Errorf("Failed to remove secondWatcher, it should have succeeded: %+v", err)
+			}
+			removed = true
+			return true
+		}
+
+		queueLen := eventManager.queue.length()
+		if queueLen != 1 {
+			t.Errorf("Failed to remove watcher, expected remaining watchers: 1, got: %d", queueLen)
+		}
+
+		if err := eventManager.RemoveWatcher(ctx, thirdWatcher); err != nil {
+			t.Errorf("Failed to remove thirdWatcher, it should have succeeded: %+v", err)
+		}
+
+		return false
+	})
+
+	if err := eventManager.Run(ctx); err != nil {
+		t.Errorf("Failed running event manager, expected success, got error: %+v", err)
+	}
+}
diff --git a/google_guest_agent/events/metadata/metadata.go b/google_guest_agent/events/metadata/metadata.go
index 9b93cb3..80a9b15 100644
--- a/google_guest_agent/events/metadata/metadata.go
+++ b/google_guest_agent/events/metadata/metadata.go
@@ -19,7 +19,6 @@ import (
 	"context"
 	"net"
 	"net/url"
-	"time"
 
 	"github.com/GoogleCloudPlatform/guest-agent/metadata"
 	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
@@ -32,11 +31,6 @@ const (
 	LongpollEvent = "metadata-watcher,longpoll"
 )
 
-var (
-	// arbitrarily defined wait duration(keeps behavioral backward compatibility).
-	retryWaitDuration = 5 * time.Second
-)
-
 // Watcher is the metadata event watcher implementation.
 type Watcher struct {
 	client         metadata.MDSClientInterface
diff --git a/google_guest_agent/events/metadata/metadata_test.go b/google_guest_agent/events/metadata/metadata_test.go
index 1b8aa5a..291f656 100644
--- a/google_guest_agent/events/metadata/metadata_test.go
+++ b/google_guest_agent/events/metadata/metadata_test.go
@@ -39,6 +39,10 @@ func (mds *mdsClient) GetKey(ctx context.Context, key string, headers map[string
 	return "", fmt.Errorf("GetKey() not yet implemented")
 }
 
+func (mds *mdsClient) GetKeyRecursive(ctx context.Context, key string) (string, error) {
+	return "", fmt.Errorf("GetKeyRecursive() not yet implemented")
+}
+
 func (mds *mdsClient) Watch(ctx context.Context) (*metadata.Descriptor, error) {
 	if !mds.disableUnknownFailure {
 		return nil, errUnknown
diff --git a/google_guest_agent/fakes/fake_mds.go b/google_guest_agent/fakes/fake_mds.go
index b6e25b1..0e8213a 100644
--- a/google_guest_agent/fakes/fake_mds.go
+++ b/google_guest_agent/fakes/fake_mds.go
@@ -37,6 +37,11 @@ func NewFakeMDSClient() *MDSClient {
 	return &MDSClient{}
 }
 
+// GetKeyRecursive implements fake GetKeyRecursive MDS method.
+func (s MDSClient) GetKeyRecursive(ctx context.Context, key string) (string, error) {
+	return "", fmt.Errorf("GetKeyRecursive() not yet implemented")
+}
+
 // GetKey implements fake GetKey MDS method.
 func (s MDSClient) GetKey(ctx context.Context, key string, headers map[string]string) (string, error) {
 	valid := `
diff --git a/google_guest_agent/instance_setup.go b/google_guest_agent/instance_setup.go
index 38c834a..6327821 100644
--- a/google_guest_agent/instance_setup.go
+++ b/google_guest_agent/instance_setup.go
@@ -25,8 +25,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/agentcrypto"
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
 	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/run"
+	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/scheduler"
 	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
 	"github.com/go-ini/ini"
 )
@@ -148,13 +150,14 @@ func agentInit(ctx context.Context) {
 			return
 		}
 
-		// The below actions require metadata to be set, so if it
-		// hasn't yet been set, wait on it here. In instances without
-		// network access, this will become an indefinite wait.
-		// TODO: split agentInit into needs-network and no-network functions.
-		for newMetadata == nil {
-			logger.Debugf("populate first time metadata...")
-			newMetadata, _ = mdsClient.Get(ctx)
+		if newMetadata == nil {
+			var err error
+			logger.Debugf("populate metadata for the first time...")
+			newMetadata, err = mdsClient.Get(ctx)
+			if err != nil {
+				logger.Errorf("Failed to reach MDS(all retries exhausted): %+v", err)
+				os.Exit(1)
+			}
 		}
 
 		// Disable overcommit accounting; e2 instances only.
@@ -204,6 +207,14 @@ func agentInit(ctx context.Context) {
 			}
 		}
 	}
+	// Schedules jobs that need to be started before notifying systemd Agent process has started.
+	// We want to generate MDS credentials as early as possible so that any process in the Guest can
+	// use them. Processes may depend on the Guest Agent at startup to ensure that the credentials are
+	// available for use. By generating the credentials before notifying the systemd, we ensure that
+	// they are generated for any process that depends on the Guest Agent.
+	if config.MDS.MTLSBootstrappingEnabled {
+		scheduler.ScheduleJobs(ctx, []scheduler.Job{agentcrypto.New()}, true)
+	}
 }
 
 func generateSSHKeys(ctx context.Context) error {
diff --git a/google_guest_agent/instance_setup_integ_test.go b/google_guest_agent/instance_setup_integ_test.go
deleted file mode 100644
index 6c343bf..0000000
--- a/google_guest_agent/instance_setup_integ_test.go
+++ /dev/null
@@ -1,208 +0,0 @@
-//  Copyright 2021 Google LLC
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-//go:build integration
-// +build integration
-
-package main
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/GoogleCloudPlatform/guest-agent/google_guest_agent/cfg"
-)
-
-const (
-	botoCfg = "/etc/boto.cfg"
-)
-
-func getConfig(t *testing.T) (*cfg.Sections, string) {
-	t.Helper()
-
-	if err := cfg.Load(nil); err != nil {
-		t.Fatalf("Failed to load configuration: %+v", err)
-	}
-
-	config, err := cfg.Get()
-	if err != nil {
-		t.Fatalf("Failed to get config: %+v", err)
-	}
-
-	if config == nil {
-		t.Fatal("cfg.Get() returned a nil config")
-	}
-
-	tempDir := filepath.Join(t.TempDir(), "test_instance_setup")
-	err := os.Mkdir(tempDir, 0755)
-	if err != nil {
-		t.Fatalf("Failed to create working dir: %+v", err)
-	}
-
-	// Configure a non-standard instance ID dir for us to play with.
-	config.Instance.InstanceIDDir = tempDir
-	config.InstanceSetup.HostKeyDir = tempDir
-
-	return config, tempDir
-}
-
-// TestInstanceSetupSSHKeys validates SSH keys are generated on first boot and not changed afterward.
-func TestInstanceSetupSSHKeys(t *testing.T) {
-	config, tempDir := getConfig(t)
-	ctx := context.Background()
-	agentInit(ctx)
-
-	if _, err := os.Stat(tempDir + "/google_instance_id"); err != nil {
-		t.Fatal("instance ID File was not created by agentInit")
-	}
-
-	dir, err := os.Open(tempDir)
-	if err != nil {
-		t.Fatal("failed to open working dir")
-	}
-	defer dir.Close()
-
-	files, err := dir.Readdirnames(0)
-	if err != nil {
-		t.Fatal("failed to read files")
-	}
-
-	var keys []string
-	for _, file := range files {
-		if strings.HasPrefix(file, "ssh_host_") {
-			keys = append(keys, file)
-		}
-	}
-
-	if len(keys) == 0 {
-		t.Fatal("instance setup didn't create SSH host keys")
-	}
-
-	// Remove one key file and run again to confirm SSH keys have not
-	// changed because the instance ID file has not changed.
-	if err := os.Remove(tempDir + "/" + keys[0]); err != nil {
-		t.Fatal("failed to remove key file")
-	}
-
-	agentInit(ctx)
-
-	if _, err := dir.Seek(0, 0); err != nil {
-		t.Fatal("failed to rewind dir for second check")
-	}
-	files2, err := dir.Readdirnames(0)
-	if err != nil {
-		t.Fatal("failed to read files")
-	}
-
-	var keys2 []string
-	for _, file := range files2 {
-		if strings.HasPrefix(file, "ssh_host_") {
-			keys2 = append(keys2, file)
-		}
-		if file == keys[0] {
-			t.Fatalf("agentInit recreated key %s", file)
-		}
-	}
-
-	if len(keys) == len(keys2) {
-		t.Fatal("agentInit recreated SSH host keys")
-	}
-}
-
-// TestInstanceSetupSSHKeysDisabled validates the config option to disable host
-// key generation is respected.
-func TestInstanceSetupSSHKeysDisabled(t *testing.T) {
-	config, tempDir := getConfig(t)
-
-	// Disable SSH host key generation.
-	config.InstanceSetup.SetHostKeys = false