Merge ~tai271828/+git/autotest-client-tests:mr-nv-performance-gpudirect-rdma into ~canonical-kernel-team/+git/autotest-client-tests:master

[Taihsiang Ho]

This merge request will create the performance test of nvidia GPUDirect technology. At this moment, there will be only one kind of testing job: peer memory testing via infinite band. It simply make sure the GPUDirect work and show the status of performance.

The job has been tested on blanka running Jammy with linux-nvidia.

[Po-Hsu Lin (cypressyew)]

+1 with tested code.

review: Approve

[Po-Hsu Lin (cypressyew)]

Applied and pushed, thanks.

diff --git a/ubuntu_performance_gpudirect_rdma/control b/ubuntu_performance_gpudirect_rdma/control
new file mode 100644
index 0000000..2e325f3
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/control
@@ -0,0 +1,13 @@
+AUTHOR = 'Taihsiang Ho <[e-mail]>'
+TIME = 'SHORT'
+NAME = 'NVIDIA GPUDirect performance test'
+TEST_TYPE = 'client'
+TEST_CLASS = 'kernel'
+TEST_CATEGORY = 'Benchmark'
+
+DOC = """
+Perform testing of NVIDIA GPUDirect performance test. At this moment, it is exercised with Infinite Band Peer Memory
+ technology.
+"""
+
+job.run_test_detail('ubuntu_performance_gpudirect_rdma', test_name='ib_peer_memory', tag='ib_peer_memory', timeout=1200)
diff --git a/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/blanka b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/blanka
new file mode 100644
index 0000000..8c7c4a8
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/blanka
@@ -0,0 +1,7 @@
+SERVER_IFACE=enp148s0
+SERVER_IP=192.168.5.1/24
+SERVER_IB_BDF=0000:4b:00.0
+
+CLIENT_IFACE=enp18s0
+CLIENT_IP=192.168.5.2/24
+CLIENT_IB_BDF=0000:ba:00.0
diff --git a/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/hot-koala b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/hot-koala
new file mode 100644
index 0000000..a76218b
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/hot-koala
@@ -0,0 +1,7 @@
+SERVER_IFACE=enp132s0
+SERVER_IP=192.168.5.1/24
+SERVER_IB_BDF=0000:84:00.0
+
+CLIENT_IFACE=ens1
+CLIENT_IP=192.168.5.2/24
+CLIENT_IB_BDF=0000:05:00.0
diff --git a/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/torchtusk b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/torchtusk
new file mode 100644
index 0000000..f8f009d
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/hosts.d/torchtusk
@@ -0,0 +1,7 @@
+SERVER_IFACE=eno33
+SERVER_IP=192.168.5.1/24
+SERVER_IB_BDF=0000:81:00.0
+
+CLIENT_IFACE=eno34
+CLIENT_IP=192.168.5.2/24
+CLIENT_IB_BDF=0000:81:00.1
diff --git a/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/nvidia-peermem-test.sh b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/nvidia-peermem-test.sh
new file mode 100755
index 0000000..c59c383
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/nvidia-peermem-test/nvidia-peermem-test.sh
@@ -0,0 +1,189 @@
+#!/bin/bash
+#
+# This is a smoke test for the kernel IB PeerDirect feature, intended
+# for monitoring Ubuntu kernel updates for regressions. We
+# don't have a unit test for just that feature, so we instead do a
+# smoke test of Nvidia's GPUDirect feature, which uses IB PeerDirect
+# underneath. This requires using the Nvidia driver stack.
+#
+# To avoid orchestrating multiple machines, we instead place 2 IB
+# devices on the same machine in separate namespaces. Running the
+# client and server in separate namespaces ensures that the traffic
+# actually flows over the IB cable between the interfaces.
+# We use ib_write_bw from the perftest package to do essentially a
+# ping test. perftest from the archive is not configured to build against
+# the (non-free) CUDA stack, so we must first rebuild it. This rebuild
+# is done in a pbuilder chroot to avoid issues w/ build-dependencies
+# installing CUDA versions that don't match the nvidia driver.
+#
+# Prerequisites:
+#   - nvidia-driver-<branch> package installed; nvidia driver loaded
+#   - nvidia-fabricmanager, if required, installed and started
+#   - 2 local IB ports connected back-to-back
+#
+# Author: dann frazier <[e-mail]>
+#
+set -e
+set -x
+
+export DEBCONF_FRONTEND="noninteractive"
+export DEBIAN_PRIORITY="critical"
+
+hostcfg="hosts.d/$HOSTNAME"
+if [ -e "$hostcfg" ]; then
+    source "$hostcfg"
+else
+    echo "ERROR: No configuration file found for $HOSTNAME" 1>&2
+    exit 1
+fi
+
+sudo_apt() {
+    sudo --preserve-env=DEBCONF_FRONTEND,DEBIAN_PRIORITY apt "$@"
+}
+
+cleanup() {
+    { [ -n "$srvpid" ] && test -d "/proc/$srvpid"; } || \
+	sudo kill "$srvpid" || /bin/true
+    [ -z "$tmpdir" ] || rm -rf "$tmpdir"
+    sudo ip addr del dev "$SERVER_IFACE" "$SERVER_IP" || /bin/true
+    sudo ip netns exec peermemclient \
+	 ip addr del dev "$CLIENT_IFACE" "$CLIENT_IP" || /bin/true
+    sudo ip netns delete peermemclient || /bin/true
+}
+trap cleanup EXIT
+
+ubuntu_mirror() {
+    local arch
+    arch="$(dpkg --print-architecture)"
+    case $arch in
+	amd64|i386)
+	    echo "http://archive.ubuntu.com/ubuntu"
+	    return
+	    ;;
+	*)
+	    echo "http://ports.ubuntu.com/ubuntu-ports"
+	    return
+	    ;;
+    esac
+}
+
+install_cuda_perftest() {
+    local release
+    local components
+    if dpkg-query -W -f '${Version}' perftest | grep -q \+cuda\.1$; then
+	# Looks like it is already build and installed
+	return
+    fi
+    release=$(lsb_release -cs)
+    components="main universe restricted multiverse"
+    # Rebuild perftest w/ CUDA support
+    sudo sed -i 's/# deb-src/deb-src/' /etc/apt/sources.list
+    sudo_apt update
+    sudo_apt build-dep -y perftest
+    sudo_apt install -y devscripts fakeroot pbuilder
+    tmpdir="$(mktemp -d)"
+    pushd "$tmpdir"
+    apt source perftest
+    pushd perftest-*
+    # There's a libnvidia-compute-<branch> package for every driver
+    # branch - each one provides a libcuda.1. dpkg-shlibdeps will
+    # generate a dependency for which package is installed at build-time.
+    # That will end up being whatever branch nvidia-cuda-dev was built for
+    # - and that may not match the driver version currently loaded. Using
+    # a mismatched libnvidia-compute/driver combo will cause ib_write_bw to
+    # error out (803 = cudaErrorSystemDriverMismatch). Override this
+    # dependency with the libnvidia-compute virtual package. We'll let
+    # apt figure out the best libnvidia-compute-<branch> package to
+    # install - it tends to pick the one that matches the installed driver.
+    echo "libcuda 1 libnvidia-compute" >> debian/shlibs.local
+    ver="$(dpkg-parsechangelog | grep ^Version: | cut -d' ' -f2)+cuda.1"
+    DEBFULLNAME="Canonical Kernel Team" \
+	       DEBEMAIL="[e-mail]" \
+	       dch -v "$ver" "Rebuild with CUDA support"
+    dpkg-buildpackage -rfakeroot -uc -us -S
+    popd
+    # We build in a pbuilder chroot instead of on the host because
+    # nvidia-cuda-dev depends may pull in nvidia package versions
+    # from branches that mismatch with the host driver branch
+    if [ ! -f "/var/cache/pbuilder/${release}.tgz" ]; then
+	sudo pbuilder create --distribution "$release" \
+	     --mirror "$(ubuntu_mirror)" \
+	     --components "$components" \
+	     --othermirror "deb $(ubuntu_mirror) ${release}-updates $components" \
+	     --basetgz "/var/cache/pbuilder/${release}.tgz"
+    fi
+    mkdir result
+    sudo sed -i 's/^export CUDA_H_PATH=.*//' /etc/pbuilderrc
+    echo "export CUDA_H_PATH=/usr/include/cuda.h" | sudo tee -a /etc/pbuilderrc
+    sudo pbuilder build --basetgz "/var/cache/pbuilder/${release}.tgz" \
+	 --extrapackages nvidia-cuda-dev \
+	 --buildresult result perftest_*cuda.1.dsc
+    sudo dpkg -i result/perftest_*cuda.1_*.deb || sudo_apt -f install -y
+    popd
+}
+
+use_cuda_needs_devid() {
+    if ib_write_bw --help | grep use_cuda=; then
+	return 0
+    fi
+    return 1
+}
+
+# Avoid dpkg lock contention
+sudo service unattended-upgrades stop || true
+
+install_cuda_perftest
+
+for ibdev in /sys/class/infiniband/*; do
+    # is this lisp?
+    bdf="$(basename "$(dirname "$(dirname "$(readlink "$ibdev")")")")"
+    case "$bdf" in
+	"$CLIENT_IB_BDF")
+	    client_ib_dev="$(basename "$ibdev")"
+	    ;;
+	"$SERVER_IB_BDF")
+	    server_ib_dev="$(basename "$ibdev")"
+	    ;;
+    esac
+done
+
+if [ -z "$client_ib_dev" ]; then
+    echo "ERROR: Could not find client infiniband device" 1>&2
+    exit 1
+fi
+if [ -z "$server_ib_dev" ]; then
+    echo "ERROR: Could not find server infiniband device" 1>&2
+    exit 1
+fi
+
+sudo rdma system set netns exclusive
+sudo ip netns add peermemclient
+sudo rdma dev set "$client_ib_dev" netns peermemclient
+sudo ip netns exec peermemclient ip link set dev lo up
+sudo ip link set netns peermemclient "$CLIENT_IFACE"
+sudo ip netns exec peermemclient ip addr add dev "$CLIENT_IFACE" "$CLIENT_IP"
+sudo ip netns exec peermemclient ip link set dev "$CLIENT_IFACE" up
+
+sudo ip addr add dev "$SERVER_IFACE" "$SERVER_IP"
+sudo ip link set dev "$SERVER_IFACE" up
+
+sudo modprobe ib_umad # bro?
+sudo modprobe nvidia-peermem
+
+sudo_apt install -y opensm
+sudo service opensm start
+
+# Sometime after focal, ib_write_bw --use_cuda began requiring a device id
+if use_cuda_needs_devid; then
+    server_use_cuda_arg="--use_cuda=0"
+    client_use_cuda_arg="--use_cuda=1"
+else
+    server_use_cuda_arg="--use_cuda"
+    client_use_cuda_arg="--use_cuda"
+fi
+sudo ib_write_bw -a -d "$server_ib_dev" "$server_use_cuda_arg" &
+srvpid=$!
+# Give server a chance to start up
+sleep 5
+sudo ip netns exec peermemclient ib_write_bw -a \
+     -d "$client_ib_dev" "${SERVER_IP%/*}" "$client_use_cuda_arg"
diff --git a/ubuntu_performance_gpudirect_rdma/ubuntu_performance_gpudirect_rdma.py b/ubuntu_performance_gpudirect_rdma/ubuntu_performance_gpudirect_rdma.py
new file mode 100644
index 0000000..a21a520
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/ubuntu_performance_gpudirect_rdma.py
@@ -0,0 +1,32 @@
+import os
+from autotest.client import test, utils
+
+p_dir = os.path.dirname(os.path.abspath(__file__))
+sh_executable = os.path.join(p_dir, "ubuntu_performance_gpudirect_rdma.sh")
+
+
+class ubuntu_performance_gpudirect_rdma(test.test):
+    version = 1
+
+    def initialize(self):
+        pass
+
+    def setup(self):
+        cmd = "{} setup".format(sh_executable)
+        utils.system(cmd)
+
+    def run_ib_peer_memory(self):
+        cmd = "{} test_ib_peer_memory".format(sh_executable)
+        utils.system(cmd)
+
+    def run_once(self, test_name):
+        if test_name == "ib_peer_memory":
+            self.run_ib_peer_memory()
+
+            print("")
+            print("{} has run.".format(test_name))
+
+        print("")
+
+    def postprocess_iteration(self):
+        pass
diff --git a/ubuntu_performance_gpudirect_rdma/ubuntu_performance_gpudirect_rdma.sh b/ubuntu_performance_gpudirect_rdma/ubuntu_performance_gpudirect_rdma.sh
new file mode 100755
index 0000000..d57d171
--- /dev/null
+++ b/ubuntu_performance_gpudirect_rdma/ubuntu_performance_gpudirect_rdma.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Exercising the NVIDIA GPU Direct RDMA performance testing on Ubuntu
+#
+
+set -eo pipefail
+
+setup() {
+    # pre-setup testing environment and necessary tools
+    # currently there is nothing practically but will be used possibly in the future.
+    echo "begin to pre-setup testing"
+}
+
+run_test() {
+    exe_dir=$(dirname "${BASH_SOURCE[0]}")
+    pushd "${exe_dir}"/nvidia-peermem-test/
+    ./nvidia-peermem-test.sh
+    popd
+}
+
+case $1 in
+    setup)
+        echo ""
+        echo "[GPUDirect RDMA] On setting up necessary test environment..."
+        echo ""
+        setup
+        echo ""
+        echo "[GPUDirect RDMA] Set up necessary test environment."
+        echo ""
+        ;;
+    test_ib_peer_memory)
+        echo ""
+        echo "[GPUDirect RDMA] On running test_ib_peer_memory..."
+        echo ""
+        run_test
+        echo ""
+        echo "[GPUDirect RDMA] Run test_ib_peer_memory."
+        echo ""
+        ;;
+esac