Merge ~ack/maas:1773150-smartctl-replace-invalid-utf8 into maas:master

[Alberto Donato]

LP:1773150 replace invalid UTF-8 chars in smartctl output

[Adam Collard (adam-collard)]

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b 1773150-smartctl-replace-invalid-utf8 lp:~ack/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas-tester/2077/consoleText
COMMIT: 478edaa00d16ba64c78873d8f7d411c14d11bdb0

review: Needs Fixing

diff --git a/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py b/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py
index a663aab..1cdb4a4 100755
--- a/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py
+++ b/src/metadataserver/builtin_scripts/testing_scripts/smartctl.py
@@ -66,7 +66,9 @@ def run_smartctl(blockdevice, args, device=None, output=False, **kwargs):
     cmd += [blockdevice]
     if output:
         print("INFO: Running command: %s" % " ".join(cmd))
-    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode()
+    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode(
+        errors="replace"
+    )
 
 
 def run_storcli(args, output=False, **kwargs):
@@ -78,7 +80,9 @@ def run_storcli(args, output=False, **kwargs):
     cmd = ["sudo", "-n", storcli] + args
     if output:
         print("INFO: Running command: %s" % " ".join(cmd))
-    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode()
+    return check_output(cmd, timeout=TIMEOUT, **kwargs).decode(
+        errors="replace"
+    )
 
 
 def make_device_name(blockdevice, device=None):
@@ -188,7 +192,7 @@ def check_SMART_support(blockdevice, device=None):
             )
             raise
         else:
-            output = e.output.decode()
+            output = e.output.decode(errors="replace")
 
     if (
         re.search(
@@ -281,7 +285,7 @@ def check_smartctl(blockdevice, device=None):
             )
             raise
         else:
-            output = e.output.decode()
+            output = e.output.decode(errors="replace")
 
     print("SUCCESS: SMART validation has PASSED for: %s" % device_name)
     if output is not None:
diff --git a/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py b/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py
index c7413af..ed0356d 100644
--- a/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py
+++ b/src/metadataserver/builtin_scripts/testing_scripts/tests/test_smartctl.py
@@ -74,6 +74,14 @@ class TestRunSmartCTL(MAASTestCase):
         )
         self.assertThat(self.mock_print, MockCalledOnce())
 
+    def test_output_invalid_utf8_replaced(self):
+        # invalid UTF-8 input
+        self.mock_check_output.return_value = b"foo\x99bar"
+        self.assertEqual(
+            "foo\ufffdbar",
+            smartctl.run_smartctl(self.blockdevice, self.args, output=True),
+        )
+
 
 class TestRunStorCLI(MAASTestCase):
     def setUp(self):