Merge ~ack/maas:1773150-smartctl-replace-invalid-utf8-3.1 into maas:3.1

[Alberto Donato]

LP:1773150 replace invalid UTF-8 chars in smartctl output

backport of 002f8c1c2d4a248b75fca33e3de29665464bf829

[Alberto Donato (ack)]

review: Approve

diff --git a/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py b/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py
index 7fba550..61c9d93 100755
--- a/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py
+++ b/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py
@@ -66,7 +66,9 @@ def run_smartctl(blockdevice, args, device=None, output=False, **kwargs):
     cmd += [blockdevice]
     if output:
         print("INFO: Running command: %s" % " ".join(cmd))
-    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode()
+    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode(
+        errors="replace"
+    )
 
 
 def run_storcli(args, output=False, **kwargs):
@@ -78,7 +80,9 @@ def run_storcli(args, output=False, **kwargs):
     cmd = ["sudo", "-n", storcli] + args
     if output:
         print("INFO: Running command: %s" % " ".join(cmd))
-    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode()
+    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode(
+        errors="replace"
+    )
 
 
 def make_device_name(blockdevice, device=None):
@@ -188,7 +192,7 @@ def check_SMART_support(blockdevice, device=None):
             )
             raise
         else:
-            output = e.output.decode()
+            output = e.output.decode(errors="replace")
 
     if (
         re.search(
@@ -281,7 +285,7 @@ def check_smartctl(blockdevice, device=None):
             )
             raise
         else:
-            output = e.output.decode()
+            output = e.output.decode(errors="replace")
 
     print("SUCCESS: SMART validation has PASSED for: %s" % device_name)
     if output is not None:
diff --git a/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py b/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py
index 6443111..8fbee1a 100644
--- a/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py
+++ b/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py
@@ -74,6 +74,14 @@ class TestRunSmartCTL(MAASTestCase):
         )
         self.assertThat(self.mock_print, MockCalledOnce())
 
+    def test_output_invalid_utf8_replaced(self):
+        # invalid UTF-8 input
+        self.mock_check_output.return_value = b"foo\x99bar"
+        self.assertEqual(
+            "foo\ufffdbar",
+            smartctl.run_smartctl(self.blockdevice, self.args, output=True),
+        )
+
 
 class TestRunStorCLI(MAASTestCase):
     def setUp(self):