Merge ~peter-sabaini/maas:add-nvme-testscript into maas:master

[Peter Sabaini]

[Lee Trager (ltrager)]

Thanks for the contribution! This needs a little bit of work before we can accept it in MAAS. In addition to the comments below this needs unit tests and to be added to the BUILTIN_SCRIPTS list in src/metadataserver/builtin_scripts/__init__.py.

review: Needs Fixing

[MAAS Lander (maas-lander)]

UNIT TESTS
-b add-nvme-testscript lp:~peter-sabaini/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/6125/console
COMMIT: 238531241f35306dc8876bf6468a72181348ac4e

review: Needs Fixing

[Blake Rouse (blake-rouse)]

Rejecting due to inactivity.

Unmerged commits

2385312... by Peter Sabaini on 2019-03-28

Add a test script for testing nvme devices.

Generate load on nvme devices for 10min, then check health via
nvme-cli

diff --git a/src/metadataserver/builtin_scripts/nvme-cli.py b/src/metadataserver/builtin_scripts/nvme-cli.py
new file mode 100644
index 0000000..1405ed4
--- /dev/null
+++ b/src/metadataserver/builtin_scripts/nvme-cli.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+#
+# nvme-cli. Generate load on nvme devices for 10min, then check health via nvme-cli
+#
+# --- Start MAAS 1.0 script metadata ---
+# name: nvme-cli.py
+# title: Check nvme health
+# description: Generate load on nvme devices, and check health via nvme-cli
+# tags: storage
+# script_type: testing
+# hardware_type: storage
+# parallel: instance
+# results:
+#  fio_errors:
+#    title: Fio errors
+#    description: Errors during fio run
+#  nvme_critical:
+#    title: NVMe critical bitvector
+#    description: NVMe critical bitvector, should be 0 for full health
+# parameters:
+#  storage: {type: storage}
+# packages: {apt: [nvme-cli, fio]}
+# destructive: true
+# --- End MAAS 1.0 script metadata ---
+
+
+import argparse
+from copy import deepcopy
+import os
+import re
+from subprocess import PIPE, Popen, STDOUT
+import sys
+
+import yaml
+
+
+FIOCMD = [
+    "sudo",
+    "-n",
+    "fio",
+    "--randrepeat=1",
+    "--ioengine=libaio",
+    "--direct=1",
+    "--gtod_reduce=1",
+    "--name=fio_test_verify",
+    "--bs=4M",
+    "--iodepth=64",
+    "--size=100%",
+    "--verify=crc32c-intel",
+    "--runtime=600s",
+    "--readwrite=randwrite",
+]
+
+NVMECMD = ["sudo", "nvme", "smart-log"]
+
+REGEX = b"err=([ 0-9]+):"
+
+
+def run_cmd(cmd):
+    """Execute `cmd` and return output or exit if error."""
+    proc = Popen(cmd, stdout=PIPE, stderr=STDOUT)
+    # Currently, we are piping stderr to STDOUT.
+    stdout, _ = proc.communicate()
+
+    # Print stdout to the console.
+    if stdout is not None:
+        print("Running command: %s\n" % " ".join(cmd))
+        print(stdout.decode())
+        print("-" * 80)
+    return stdout, proc.returncode
+
+
+def run_fio_test(result_path):
+    """Run fio for the given type of test specified by `cmd`."""
+    stdout, returncode = run_cmd(FIOCMD)
+    if returncode != 0:
+        sys.exit(proc.returncode)
+    if result_path is not None:
+        # Parse the results for the desired information and
+        # then wrtie this to the results file.
+        match = re.search(REGEX, stdout)
+        if match is None:
+            print("Warning: results could not be found.")
+        return match
+
+
+def run_nvme_smartlog(storage):
+    stdout, returncode = run_cmd(NVMECMD + [storage])
+    return stdout, returncode
+
+
+def run_nvme_critical(storage):
+    stdout, returncode = run_nvme_smartlog(storage)
+    if returncode == 0:
+        match = re.search(b"critical_warning.*: ([0-9]+)", stdout)
+        if match is None:
+            print("Warning: nvme results could not be found.")
+        return match
+    else:
+        sys.exit(proc.returncode)
+
+
+def write_results(status, fio_errors, nvme_crit, result_path):
+    results = {"status": status, "results": {"fio_errors": fio_errors, "nvme_critical": nvme_crit}}
+    with open(result_path, "w") as results_file:
+        yaml.safe_dump(results, results_file)
+
+
+def run_nvme(storage):
+    """Execute nvme tests for supplied storage device.
+
+    Performs random write tests with verification, then checks nvme health
+    """
+    result_path = os.environ.get("RESULT_PATH")
+    _, returncode = run_nvme_smartlog(storage)
+    if returncode != 0:  # this is not an NVMe device
+        write_results("passed", 0, 0, result_path)
+        return
+
+    FIOCMD.append("--filename=%s" % storage)
+    random_write_match = run_fio_test(result_path)
+    fio_errors = int(random_write_match.group(1).decode())
+    nvme_match = run_nvme_critical(storage)
+    nvme_crit = int(nvme_match.group(1).decode())
+    if fio_errors == 0 and nvme_crit == 0:
+        status = "passed"
+    else:
+        status = "failed"
+    write_results(status, fio_errors, nvme_crit, result_path)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="NVMe Hardware Testing.")
+    parser.add_argument("--storage", dest="storage", help="path to storage device you want to test. e.g. /dev/nvme0")
+    args = parser.parse_args()
+    sys.exit(run_nvme(args.storage))