Merge ~xavpaice/charm-hw-health:LP1838562 into ~nagios-charmers/charm-hw-health:master

[Xav Paice]

Update ipmi cron job to cater for stale lockfile

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Peter Sabaini (peter-sabaini)]

Some comments inline, otherwise lgtm

review: Approve

[Xav Paice (xavpaice)]

Updated the change to add the file paths in the error messages trapped.

[Stuart Bishop (stub)]

Looks good

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 6c29dde4a3f29fc7137619dea3cfe399f196dd79

diff --git a/src/files/ipmi/cron_ipmi_sensors.py b/src/files/ipmi/cron_ipmi_sensors.py
index a9538af..a11f9fb 100644
--- a/src/files/ipmi/cron_ipmi_sensors.py
+++ b/src/files/ipmi/cron_ipmi_sensors.py
@@ -17,30 +17,41 @@ NAGIOS_ERRORS = {
 
 
 def gather_metrics():
-    # a child is already running
+    # Check if a PID file exists
     if os.path.exists(CHECK_IPMI_PID):
-        return
+        # is the PID valid?
+        with open(CHECK_IPMI_PID, 'r') as fd:
+            PID = fd.read()
+        if PID not in os.listdir('/proc'):
+            # PID file is invalid, remove it
+            os.remove(CHECK_IPMI_PID)
+        else:
+            return
 
     try:
         with open(CHECK_IPMI_PID, 'w') as fd:
             fd.write(str(os.getpid()))
-
-        cmdline = [CMD]
-        if len(sys.argv) > 1:
-            cmdline.extend(sys.argv[1:])
-
+    except IOError as e:
+        # unable to write PID file, can't lock
+        print("Cannot write lock file {}, error {}".format(CHECK_IPMI_PID, e))
+        sys.exit(1)
+
+    cmdline = [CMD]
+    if len(sys.argv) > 1:
+        cmdline.extend(sys.argv[1:])
+    try:
         output = subprocess.check_output(cmdline)
-        with open(TMP_OUTPUT_FILE, 'w') as fd:
-            fd.write(output.decode(errors='ignore'))
-        os.rename(TMP_OUTPUT_FILE, OUTPUT_FILE)
     except subprocess.CalledProcessError as error:
         output = error.stdout.decode(errors='ignore')
         with open(TMP_OUTPUT_FILE, 'w') as fd:
             fd.write('{}: {}'.format(NAGIOS_ERRORS[error.returncode], output))
-        os.rename(TMP_OUTPUT_FILE, OUTPUT_FILE)
-    except PermissionError as error:
-        with (OUTPUT_FILE, 'w') as fd:
-            fd.write('UNKNOWN: {}'.format(error))
+    try:
+        with open(TMP_OUTPUT_FILE, 'w') as fd:
+            fd.write(output)
+    except IOError as e:
+        print("Cannot write output file {}, error {}".format(TMP_OUTPUT_FILE, e))
+        sys.exit(1)
+    os.rename(TMP_OUTPUT_FILE, OUTPUT_FILE)
 
     # remove pid reference
     os.remove(CHECK_IPMI_PID)