Merge ~mitchdz/ubuntu/+source/ec2-hibinit-agent:mitch/xenial-imdsv2 into ubuntu/+source/ec2-hibinit-agent:ubuntu/xenial-devel

[Mitchell Dzurick]

PPA - https://launchpad.net/~mitchdz/+archive/ubuntu/ec2-hibinit-agent-imdsv2

92c89f4... by Mitchell Dzurick on 2023-11-22

debian/hibinit-resume: Add extra steps around swapoff to avoid OOM errors.

4054ec8... by Mitchell Dzurick on 2023-11-28

d/p/lp1968805-Swapon-with-maximum-priority-before-hibernation.patch: Swapon with maximum priority right before hibernation.

13f7bf9... by Mitchell Dzurick on 2023-11-28

d/p/{lp1941785,lp2023924}*: add IMDSv2, fix quotes bug created from IMDSv2 patch

diff --git a/debian/changelog b/debian/changelog
index b366dca..b4f95a5 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,19 @@
+ec2-hibinit-agent (1.0.0-0ubuntu4~16.04.5) xenial; urgency=medium
+
+  * d/p/lp2023924-remove-quotes-from-state-dir.patch: fixes an issue where a
+    weird directory would be created in root (LP: #2023924)
+  * d/p/lp1941785-Add-support-for-IMDSv2.patch: allow hibernation of
+    AWS EC2 instances with IMDSv2 (LP: #1941785)
+  * Swapon with maximum priority right before hibernation. This resolves
+    swapfile priority issues with additional or multiple swapfiles enabled.
+    (LP: #1968805)
+    - d/p/lp1968805-Swapon-with-maximum-priority-before-hibernation.patch
+  * debian/hibinit-resume: Add extra steps around swapoff to avoid OOM errors.
+    Also work around xen-netfront not resuming properly.
+    Thanks to Francis Ginther for the initial patch (LP: #1863242, #1864041)
+
+ -- Mitchell Dzurick <[e-mail]>  Tue, 21 Nov 2023 15:31:14 -0700
+
 ec2-hibinit-agent (1.0.0-0ubuntu4~16.04.4) xenial; urgency=medium
 
   * Always set resume device by PARTUUID instead of by device name.
diff --git a/debian/ec2-hibinit-agent.install b/debian/ec2-hibinit-agent.install
index b140d28..5b1b272 100755
--- a/debian/ec2-hibinit-agent.install
+++ b/debian/ec2-hibinit-agent.install
@@ -5,5 +5,6 @@ debian/ec2-hibinit-agent-ignore-powerkey.conf usr/lib/systemd/logind.conf.d/
 debian/hibinit-agent.system-sleep => lib/systemd/system-sleep/hibinit-agent
 debian/hibinit-power etc/acpi/events
 debian/hibinit-power.sh etc/acpi/actions
+debian/hibinit-resume usr/lib/ec2-hibinit-agent
 etc/hibinit-config.cfg
 agent/* usr/bin/
diff --git a/debian/hibinit-agent.system-sleep b/debian/hibinit-agent.system-sleep
index 914161b..62db7bd 100755
--- a/debian/hibinit-agent.system-sleep
+++ b/debian/hibinit-agent.system-sleep
@@ -4,8 +4,7 @@ set -e
 if [ "$2" = "hibernate" ] || [ "$2" = "hybrid-sleep" ]; then
     case "$1" in
         post)
-            systemctl restart --no-block systemd-networkd
-            systemd-run --no-block --service-type=oneshot swapoff /swap-hibinit
+            systemd-run --no-block --service-type=oneshot /usr/lib/ec2-hibinit-agent/hibinit-resume
             ;;
     esac
 fi
diff --git a/debian/hibinit-resume b/debian/hibinit-resume
new file mode 100755
index 0000000..e83fe2c
--- /dev/null
+++ b/debian/hibinit-resume
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Resume from hibernation on EC2 and apply workarounds for various issues
+#
+# Copyright (C) 2020 Canonical Ltd.
+#
+
+set -e
+
+# Starting hibinit resume actions
+echo "hibinit-resume: begin" >> /var/log/hibinit-resume.log
+
+# Give some time to make sure the system is fully up and running, see LP: #1864041
+sleep 2
+
+# Make sure the network is properly resumed
+echo "restarting network" >> /var/log/hibinit-resume.log
+if grep -q xen_netfront /proc/modules; then
+    # Xen netfront workaround LP: #1864041
+    modprobe -r xen_netfront
+    modprobe xen_netfront
+fi
+systemctl restart --no-block systemd-networkd
+
+# Temporarily set memory allocation policy to "always overcommit". With
+# the default overcommit policy some applications might experience
+# out-of-memory errors. Setting the policy to "always overcommit" seems
+# to be a safe way to prevent these errors.
+OVERCOMMIT_POLICY=$(cat /proc/sys/vm/overcommit_memory)
+echo 1 > /proc/sys/vm/overcommit_memory
+# Start swapoff and make sure swap file is gone
+while :; do
+    echo "starting swapoff" >> /var/log/hibinit-resume.log
+    swapoff /swap-hibinit
+    if grep -q swap-hibinit /proc/swaps; then
+        echo "swap still active" >> /var/log/hibinit-resume.log
+        sleep 10
+    else
+        break
+    fi
+done
+echo "swap disabled" >> /var/log/hibinit-resume.log
+cat /proc/swaps >> /var/log/hibinit-resume.log
+# Restore previou overcommit policy
+echo "$OVERCOMMIT_POLICY" > /proc/sys/vm/overcommit_memory
+
+# All done
+echo "hibinit-resume: completed " >> /var/log/hibinit-resume.log
diff --git a/debian/patches/lp1941785-Add-support-for-IMDSv2.patch b/debian/patches/lp1941785-Add-support-for-IMDSv2.patch
new file mode 100644
index 0000000..6855eb2
--- /dev/null
+++ b/debian/patches/lp1941785-Add-support-for-IMDSv2.patch
@@ -0,0 +1,161 @@
+Description: Add support for IMDSv2
+Author: Frederick Lefebvre <[e-mail]>
+Origin: backport, https://github.com/aws/amazon-ec2-hibinit-agent/commit/9d9bca5c61fa9256289e68c88bd3747af2f62e28
+Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/ec2-hibinit-agent/+bug/1941785
+Last-Update: 2023-05-16
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/agent/hibinit-agent
++++ b/agent/hibinit-agent
+@@ -18,6 +18,7 @@
+ import sys
+ import syslog
+ import math
++import requests
+ from subprocess import check_call, check_output, STDOUT
+ from threading import Thread
+ from math import ceil
+@@ -25,11 +26,6 @@
+ 
+ 
+ try:
+-    from urllib.request import urlopen, Request
+-except ImportError:
+-    from urllib2 import urlopen, Request, HTTPError
+-
+-try:
+     from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+ except:
+     from configparser import ConfigParser, NoSectionError, NoOptionError
+@@ -41,7 +37,12 @@
+ log_to_syslog = True
+ log_to_stderr = True
+ SWAP_FILE = '/swap-hibinit'
+-URL = "http://169.254.169.254/latest/meta-data/hibernation/configured"
++
++DEFAULT_STATE_DIR = '/var/lib/hibinit-agent'
++HIB_ENABLED_FILE = "hibernation-enabled"
++IMDS_BASEURL = 'http://169.254.169.254'
++IMDS_API_TOKEN_PATH = 'latest/api/token'
++IMDS_SPOT_ACTION_PATH = 'latest/meta-data/hibernation/configured'
+ 
+ def log(message):
+     if log_to_syslog:
+@@ -314,6 +315,9 @@
+             get_int('swap', 'percentage-of-ram'), args.swap_ram_percentage, 100)
+         self.swap_mb = self.merge(
+             get_int('swap', 'target-size-mb'), args.swap_target_size_mb, 4000)
++        self.state_dir = get('core', 'state-dir')
++        if self.state_dir is None:
++            self.state_dir = DEFAULT_STATE_DIR
+ 
+ 
+     def merge(self, cf_value, arg_value, def_val):
+@@ -337,31 +341,55 @@
+     def __str__(self):
+         return str(self.__dict__)
+ 
+-def hibernationEnabled():
+-     """Returns a boolean indicating whether hibernation is enabled or not."""
+-     response = None
+-     try:
+-         response = urlopen(URL)
+-         data = response.read()
+-         if data.lower() in ('false', b'false'):
+-             return False
+-     except:
+-         return False
+-     finally:
+-         if response:
+-             response.close()
+-     return True
++def get_imds_token(seconds=21600):
++    """ Get a token to access instance metadata. """
++    log("Requesting new IMDSv2 token.")
++    request_header = {'X-aws-ec2-metadata-token-ttl-seconds': str(seconds)}
++    token_url = '{}/{}'.format(IMDS_BASEURL, IMDS_API_TOKEN_PATH)
++    response = requests.put(token_url, headers=request_header)
++    response.close()
++    if response.status_code != 200:
++        return None
++
++    return response.text
++
++def create_state_dir(state_dir):
++    """ Create agent run dir if it doesn't exists."""
++    if not os.path.isdir(state_dir):
++        os.makedirs(state_dir)
++
++def hibernation_enabled(state_dir):
++    """Returns a boolean indicating whether hibernation is enabled or not.
++    Hibernation can't be enabled/disabled the instance launch. If we find
++    hibernation to be enabled, we create a semephore file so that we don't
++    have to probe IMDS again. That is useful when a instance is rebooted
++    after/if the IMDS http endpoint has been disabled.
++    """
++    hib_sem_file = os.path.join(state_dir, HIB_ENABLED_FILE)
++    if os.path.isfile(hib_sem_file):
++        log("Found {!r}, configuring hibernation".format(hib_sem_file))
++        return True
++
++    imds_token = get_imds_token()
++    if imds_token is None:
++        # IMDS http endpoint is disabled
++        return False
++
++    request_header = {'X-aws-ec2-metadata-token': imds_token}
++    response = requests.get("{}/{}".format(IMDS_BASEURL, IMDS_SPOT_ACTION_PATH),
++                 headers=request_header)
++    response.close()
++    if response.status_code != 200 or response.text.lower() == "false":
++        return False
++
++    log("Hibernation Configured Flag found")
++    os.mknod(hib_sem_file)
++
++    return True
++
+ 
+ def main():
+     
+-    if not hibernationEnabled():
+-        log("Instance Launch has not enabled Hibernation Configured Flag. hibinit-agent exiting!!")
+-        exit(0)
+-    # Validate if disk space>total RAM 
+-    ram_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
+-    if get_rootfs_size()<=(math.ceil(float(ram_bytes)/(1024*1024*1024))):
+-        log("Insufficient disk space. Cannot create setup for hibernation. Please allocate a larger root device")
+-        exit(1)
+     # Parse arguments
+     parser = argparse.ArgumentParser(description="An EC2 background process that creates a setup for instance hibernation "
+                                                  "at instance launch and also registers ACPI sleep event/actions")
+@@ -388,6 +416,17 @@
+     log_to_syslog = config.log_to_syslog
+ 
+     log("Effective config: %s" % config)
++    create_state_dir(config.state_dir)
++
++    # Let's first check if we even need to run
++    if not hibernation_enabled(config.state_dir):
++        log("Instance Launch has not enabled Hibernation Configured Flag. hibinit-agent exiting!!")
++        exit(0)
++    # Validate if disk space>total RAM
++    ram_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
++    if get_rootfs_size()<=(math.ceil(float(ram_bytes)/(1024*1024*1024))):
++        log("Insufficient disk space. Cannot create setup for hibernation. Please allocate a larger root device")
++        exit(1)
+ 
+     target_swap_size = config.swap_mb * 1024 * 1024
+     swap_percentage_size = ram_bytes * config.swap_percentage // 100
+--- a/etc/hibinit-config.cfg
++++ b/etc/hibinit-config.cfg
+@@ -11,6 +11,9 @@
+ # filesystems.
+ touch-swap = False
+ 
++# Location where to create any state files
++state-dir = "/var/lib/hibinit-agent"
++
+ [swap]
+ # If there's no swap then we create it to be equal to the specified
+ # percentage of RAM or to the target size, whichever is greater
diff --git a/debian/patches/lp1968805-Swapon-with-maximum-priority-before-hibernation.patch b/debian/patches/lp1968805-Swapon-with-maximum-priority-before-hibernation.patch
new file mode 100644
index 0000000..5ed11c1
--- /dev/null
+++ b/debian/patches/lp1968805-Swapon-with-maximum-priority-before-hibernation.patch
@@ -0,0 +1,63 @@
+Author: Matthew Ruffell <[e-mail]>
+Date: Wed 13 Apr 2022 15:39:15 +1200
+Description: 
+
+    Swapon with maximum priority before hibernation
+    
+    Since Focal, on systems with additional or multiple swapfiles
+    configured, swapon on /swap-hibinit will be added with the next
+    lowest priority for the system.
+    
+    For example, if we have /swapfile configured with a default priority
+    of -2, swapon /swap-hibinit will be added with priority -3.
+    
+    $ sudo swapon /swapfile
+    $ swapon --show
+    NAME      TYPE SIZE USED PRIO
+    /swapfile file   4G   0B   -2
+    $ sudo swapon /swap-hibinit
+    $ swapon --show
+    NAME          TYPE SIZE USED PRIO
+    /swapfile     file   4G   0B   -2
+    /swap-hibinit file 3.9G   0B   -3
+    
+    This causes hibernation to fail, since hibernation targets the
+    highest priority swapfile, which is no longer /swap-hibinit, and the
+    kernel command line is specifically configured to resume from the
+    UUID and offset for /swap-hibinit.
+    
+    The result is that EC2 console says "Stopping" for 20 minutes before
+    the instance is force stopped.
+    
+    Fix this by setting /swap-hibinit to maximum possible priority,
+    32767, as we swapon right before hibernation takes place.
+
+Bug-Ubuntu: https://bugs.launchpad.net/bugs/1968805
+Forwarded: yes, https://github.com/aws/amazon-ec2-hibinit-agent/commit/a2303d269610a6e7415c5045766da605eaa7e30f
+Last-Update: 2022-04-13
+
+Index: ec2-hibinit-agent-1.0.0/acpid.sleep.sh
+===================================================================
+--- ec2-hibinit-agent-1.0.0.orig/acpid.sleep.sh	2022-04-13 15:55:57.642062380 +1200
++++ ec2-hibinit-agent-1.0.0/acpid.sleep.sh	2022-04-13 15:55:57.642062380 +1200
+@@ -2,6 +2,11 @@
+ 
+ #PATH=/sbin:/bin:/usr/bin
+ 
++# Hibernation selects the swapfile with highest priority. Since there may be
++# other swapfiles configured, ensure /swap-hibinit is selected as hibernation
++# target by setting to maximum priority. See LP#1968805.
++swap_priority=32767
++
+ do_hibernate() {
+     if [ -d /run/systemd/system ]; then
+         systemctl hibernate
+@@ -14,7 +19,7 @@
+ 
+ case "$2" in
+     SBTN)
+-        swapon /swap-hibinit && do_hibernate
++        swapon --priority=$swap_priority /swap-hibinit && do_hibernate
+         ;;
+     *)
+         logger "ACPI action undefined: $2" ;;
diff --git a/debian/patches/lp2023924-remove-quotes-from-state-dir.patch b/debian/patches/lp2023924-remove-quotes-from-state-dir.patch
new file mode 100644
index 0000000..d178d21
--- /dev/null
+++ b/debian/patches/lp2023924-remove-quotes-from-state-dir.patch
@@ -0,0 +1,19 @@
+Description: Remove quotes in config file for state_dir value
+It creates strange directory /"
+Author: Anchal Agarwal <[e-mail]>
+Origin: backport, https://github.com/aws/amazon-ec2-hibinit-agent/commit/d87d6365009f1663247fd8595b487c59d9b121f1
+Bug-Ubuntu: https://bugs.launchpad.net/ubuntu/+source/ec2-hibinit-agent/+bug/2023924
+Last-Update: 2023-06-20
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/etc/hibinit-config.cfg
++++ b/etc/hibinit-config.cfg
+@@ -12,7 +12,7 @@
+ touch-swap = False
+ 
+ # Location where to create any state files
+-state-dir = "/var/lib/hibinit-agent"
++state-dir = /var/lib/hibinit-agent
+ 
+ [swap]
+ # If there's no swap then we create it to be equal to the specified
diff --git a/debian/patches/series b/debian/patches/series
index 136d930..1555533 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -7,3 +7,6 @@ detect-hibernate-cmd-by-default.patch
 0008-Detect-disabled-hibernation-with-Python-3-s-urllib-t.patch
 0008-Always-set-resume-device-by-PARTUUID-instead-of-by-d.patch
 0010-Update-grub-configuration-when-it-needs-an-update.patch
+lp1968805-Swapon-with-maximum-priority-before-hibernation.patch
+lp1941785-Add-support-for-IMDSv2.patch
+lp2023924-remove-quotes-from-state-dir.patch