Merge ~canonical-kernel-team/+git/autotest-client-tests:phlin/add-ubuntu_kselftests_net into ~canonical-kernel-team/+git/autotest-client-tests:master

[Po-Hsu Lin]

After moving ftrace out into ubuntu_kselftests_ftrace, net test is now
the biggest test in ubuntu_kernel_selftests. It can take up to 7h25m
on a Mantic Starfive VM (the whole test will take 8h9m)

By moving this test out it will:
  * Shorten the time we need if we need to rerun the test.
  * More flexibility for test suite selection (for CVE respins).
  * Give us more room to cover more tests in ubuntu_kernel_selftests,
    without the need to bump the timeout again.
  * If there is anything goes wrong in net category, we can still
    finish other tests in ubuntu_kernel_selftests.

The trade off is it will take extra provision time.

[Po-Hsu Lin (cypressyew)]

After merging this and it's running OK, we can proceed to remove the net test in https://code.launchpad.net/~canonical-kernel-team/+git/autotest-client-tests/+merge/456774

[Po-Hsu Lin (cypressyew)]

Inline comment replied, thanks for reviewing this. I will update the commit.

[Po-Hsu Lin (cypressyew)]

New changes include:
  * Adding missing helper.mk
  * Fixing erroneous if statement for fcnal-test.sh test check
  * Adding missing kernel-wedge package
  * Fix a mis-indented else statement in control file.

Patch tested with B/F/J/L/M.

[Magali Lemes do Sacramento (magalilemes)]

LGTM!

review: Approve

[Sean Feole (sfeole)]

+1 , please see : https://kernel.ubuntu.com/gitea/tnt/ckct/pulls/160

review: Approve

[Po-Hsu Lin (cypressyew)]

Applied and pushed, thanks!

diff --git a/ubuntu_kselftests_net/blacklist.trusty b/ubuntu_kselftests_net/blacklist.trusty
new file mode 100644
index 0000000..4682f80
--- /dev/null
+++ b/ubuntu_kselftests_net/blacklist.trusty
@@ -0,0 +1 @@
+We don't run net test on Trusty, just ['setup','breakpoints','cpu-hotplug','efivarfs','ipc','mount','powerpc','ptrace'].
diff --git a/ubuntu_kselftests_net/control b/ubuntu_kselftests_net/control
new file mode 100644
index 0000000..23031df
--- /dev/null
+++ b/ubuntu_kselftests_net/control
@@ -0,0 +1,47 @@
+AUTHOR = "Ubuntu"
+NAME = 'ubuntu_kselftests_net'
+CRITERIA = """
+Uses built-in kernel repository self tests for net.
+"""
+SUITE = "None"
+TIME = "SHORT"
+TEST_CLASS = 'kernel'
+TEST_CATEGORY = 'Functional'
+TEST_TYPE = "client"
+DOC = ""
+
+
+categories = ['net']
+arch = platform.machine()
+arch_scale = 1
+# Scale timeouts by 2 for riscv64, some tests timeout due to lack of
+# timeout, despite progressing fine
+if arch in ['riscv64']:
+    arch_scale = 2
+
+result = job.run_test_detail(NAME, test_name='setup', tag='setup', timeout=arch_scale*60*45)
+if result == 'GOOD':
+    for category in categories:
+        build = '{}-build'.format(category)
+        result = job.run_test_detail(NAME, test_name=build, tag=build, timeout=arch_scale*60*60)
+        if result == 'ERROR':
+            print("ERROR: test suite '{}' failed to build, skipping all the sub tests".format(category))
+            continue
+        mk_helper = os.path.join(job.testdir, NAME, 'helper.mk')
+        dir_root = os.path.join(job.bindir, 'tmp', NAME, 'src', 'linux/tools/testing/selftests/')
+        dir_src = os.path.join(dir_root, category)
+        mk_src = os.path.join(dir_src, 'Makefile')
+        os.chdir(dir_src)
+        if os.path.isfile(mk_src):
+            cmd = 'make -f {} -f {} gettests'.format(mk_src, mk_helper)
+            tests = utils.system_output(cmd).split()
+            for item in tests:
+                timeout_threshold = arch_scale*60*45
+                if item == 'fcnal-test.sh':
+                    timeout_threshold = arch_scale*60*75  # lp:2006499, lp:2012016
+                test = "{}:{}".format(category, item)
+                job.run_test_detail(NAME, test_name=test, tag=test, timeout=timeout_threshold)
+else:
+    print("ERROR: test failed to build, skipping all the sub tests")
+
+# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kselftests_net/control.ubuntu.xenial b/ubuntu_kselftests_net/control.ubuntu.xenial
new file mode 100644
index 0000000..37dec82
--- /dev/null
+++ b/ubuntu_kselftests_net/control.ubuntu.xenial
@@ -0,0 +1,33 @@
+AUTHOR = "Ubuntu"
+NAME = 'ubuntu_kselftests_net'
+CRITERIA = """
+Uses built-in kernel repository self tests for net.
+"""
+SUITE = "None"
+TIME = "SHORT"
+TEST_CLASS = 'kernel'
+TEST_CATEGORY = 'Functional'
+TEST_TYPE = "client"
+DOC = ""
+
+
+categories = ['net']
+
+result = job.run_test_detail(NAME, test_name='setup', tag='setup', timeout=60*30)
+if result == 'ERROR':
+    print("ERROR: test failed to build, skipping all the sub tests")
+else:
+    for category in categories:
+        build = '{}-build'.format(category)
+        job.run_test_detail(NAME, test_name=build, tag=build, timeout=60*10)
+        mk_helper = os.path.join(job.testdir, NAME, 'helper.mk')
+        dir_src = os.path.join(job.bindir, 'tmp', NAME, 'src', 'linux/tools/testing/selftests/', category)
+        mk_src = os.path.join(dir_src, 'Makefile')
+        os.chdir(dir_src)
+        cmd = 'make -f {} -f {} gettests'.format(mk_helper, mk_src)
+        tests = utils.system_output(cmd).split()
+        for item in tests:
+            test = "{}:{}".format(category, os.path.basename(item))
+            job.run_test_detail(NAME, test_name=test, tag=test, timeout=60*30)
+
+# vi:set ts=4 sw=4 expandtab syntax=python:
diff --git a/ubuntu_kselftests_net/helper.mk b/ubuntu_kselftests_net/helper.mk
new file mode 100755
index 0000000..ab3bb36
--- /dev/null
+++ b/ubuntu_kselftests_net/helper.mk
@@ -0,0 +1,5 @@
+gettests:
+	@echo '$(notdir $(TEST_GEN_PROGS)) $(notdir $(TEST_CUSTOM_PROGS)) $(notdir $(TEST_PROGS))'
+
+getsubdirs:
+	@echo '$(SUB_DIRS)'
diff --git a/ubuntu_kselftests_net/ubuntu_kselftests_net.py b/ubuntu_kselftests_net/ubuntu_kselftests_net.py
new file mode 100644
index 0000000..766c904
--- /dev/null
+++ b/ubuntu_kselftests_net/ubuntu_kselftests_net.py
@@ -0,0 +1,174 @@
+#
+#
+import os
+import platform
+import re
+from autotest.client                        import test, utils
+from autotest.client.shared                 import error
+
+class ubuntu_kselftests_net(test.test):
+    version = 1
+
+    def install_required_pkgs(self):
+        '''Function to install necessary packages.'''
+        pkgs = [
+            'build-essential',
+            'docutils-common',  # For bpf test build
+            'ethtool',          # For net:udpgro_fwd.sh
+            'iptables',         # For net:ip_defrag.sh
+            'jq',               # For net:altnames.sh
+            'kernel-wedge',     # For "fakeroot debian/rules clean"
+            'libfuse-dev',      # For net:reuseport_bpf_numa
+            'libnuma-dev',      # For net:reuseport_bpf_numa
+            'libssl-dev',       # For net:tcp_mmap
+            'netsniff-ng',      # For net:altnames.sh
+            'net-tools',        # For net:rtnetlink.sh
+            'pkg-config',
+            'uuid-runtime',     # For net:rtnetlink.sh
+        ]
+        # For net:fib_tests.sh
+        if not self.series in ['trusty', 'xenial', 'bionic']:
+            pkgs.append('socat')
+        if not self.arch == 's390x':
+            if not self.series in ['trusty', 'xenial', 'bionic', 'focal', 'jammy']:
+                # With recent kernels BPF requires lld (LLVM-based linker) to
+                # build the corresponding kernel selftests, so make sure this
+                # package is installed (in the releases where it is available)
+                # lld is not available for s390x
+                pkgs.append('lld')
+        gcc = 'gcc' if self.arch in ['ppc64le', 'aarch64', 's390x', 'riscv64'] else 'gcc-multilib'
+        pkgs.append(gcc)
+
+        # The modules-extra is required for net:rtnetlink.sh test
+        if any(x in self.flavour for x in ['aws', 'azure', 'gcp', 'gke', 'ibm', 'oracle']):
+            if not (self.flavour == 'aws' and self.series == 'trusty'):
+                pkgs.append('linux-modules-extra-' + platform.uname()[2])
+
+        if self.kv >= 415:
+            # extra packages for building bpf tests, which is required for some tests in net
+            pkgs.extend(['libcap-dev', 'libelf-dev'])
+            if self.kv == 504:
+                # special case for B-5.4 (lp:1882559) / B-5.3 (lp:1845860)
+                # clang on F is clang-10 but we need clang-9 (see commit 95f91d59642)
+                # clang on E is clang-9, so it's ok to just check kv here
+                pkgs.extend(['clang-9', 'llvm-9'])
+            else:
+                pkgs.extend(['clang', 'llvm'])
+
+        cmd = 'yes "" | DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes ' + ' '.join(pkgs)
+        utils.system_output(cmd, retain_output=True)
+
+    def initialize(self):
+        self.arch = platform.processor()
+        self.flavour = re.split('-\d*-', platform.uname()[2])[-1]
+        try:
+            self.series = platform.dist()[2]
+        except AttributeError:
+            import distro
+            self.series = distro.codename()
+        self.kv = platform.release().split(".")[:2]
+        self.kv = int(self.kv[0]) * 100 + int(self.kv[1])
+
+    def download(self):
+        '''Function to download kernel source.'''
+        cmd = "dpkg -S /lib/modules/" + platform.release() + "/kernel | cut -d: -f 1 | cut -d, -f 1"
+        pkg = os.popen(cmd).readlines()[0].strip()
+        utils.system("apt-get source --download-only " + pkg)
+
+    def extract(self):
+        '''Function to extract kernel source.'''
+        os.system("rm -rf linux/")
+        utils.system("dpkg-source -x linux*dsc linux")
+
+    def setup(self):
+        '''Function to setup the test environment.'''
+        self.install_required_pkgs()
+        self.job.require_gcc()
+        os.chdir(self.srcdir)
+
+        # Use a local repo for manual testing. If it does not exist, then clone from the master
+        # repository.
+        #
+        if not os.path.exists('linux'):
+            self.download()
+            self.extract()
+
+            # clean source tree so changes from debian.foo/reconstruct
+            # (e.g. deleting files) are applied
+            os.chdir('linux')
+            cmd = 'fakeroot debian/rules clean'
+            utils.system(cmd)
+            os.chdir(self.srcdir)
+
+            # net/txtimestamp.sh is very fragile, disable it
+            #
+            fn = 'linux/tools/testing/selftests/net/Makefile'
+            if os.path.exists(fn):
+                cmd = 'sed -i "/^TEST_PROGS += txtimestamp.sh$/d" ' + fn
+                utils.system(cmd)
+
+            # Build header first (LP: #2031400)
+            if not self.series in ['trusty', 'xenial', 'bionic', 'focal']:
+                cmd = "make -C linux/ headers"
+                utils.system_output(cmd, retain_output=True)
+
+
+    def run_once(self, test_name):
+        if test_name == 'setup':
+            return
+        if test_name.endswith('-build'):
+            os.chdir(self.srcdir)
+            if "net" in test_name:
+                cmds = []
+                cmds.append("sh -c 'echo 1 > /proc/sys/net/ipv4/conf/all/accept_local'")
+                # The net benchmarching tests (e.g. udpgso) can fail when
+                # optmem limit is reached.
+                # https://www.kernel.org/doc/html/latest/networking/msg_zerocopy.html#transmission
+                # LP #1960907
+                cmds.append("sh -c 'echo 2048000 > /proc/sys/net/core/optmem_max'")
+                for cmd in cmds:
+                    utils.system(cmd)
+
+                if self.kv >= 415:
+                    # net selftests use a module built by bpf selftests, bpf is available since bionic kernel
+                    if self.kv == 504:
+                        os.environ["CLANG"] = "clang-9"
+                        os.environ["LLC"] = "llc-9"
+                        os.environ["LLVM_OBJCOPY"] = "llvm-objcopy-9"
+                        os.environ["LLVM_READELF"] = "llvm-readelf-9"
+                    cmd = "make -C linux/tools/testing/selftests TARGETS=bpf SKIP_TARGETS= KDIR=/usr/src/linux-headers-{}".format(platform.release())
+                    # keep running selftests/net, even if selftests/bpf build fails
+                    utils.system(cmd, ignore_status=True)
+            cmd = "make -C linux/tools/testing/selftests TARGETS={}".format(test_name.replace('-build', ''))
+            utils.system_output(cmd, retain_output=True)
+            return
+
+        category = test_name.split(':')[0]
+        sub_test = test_name.split(':')[1]
+        dir_root = os.path.join(self.srcdir, 'linux', 'tools', 'testing', 'selftests')
+        os.chdir(dir_root)
+        cmd = "make run_tests -C {} TEST_PROGS={} TEST_GEN_PROGS='' TEST_CUSTOM_PROGS=''".format(category, sub_test)
+        result = utils.system_output(cmd, retain_output=True)
+
+        # The output of test_bpf.sh / test_blackhole_dev.sh test will be in the dmesg
+        kernel_module_tests = {'test_bpf.sh': 'CONFIG_TEST_BPF',
+                               'test_blackhole_dev.sh': 'CONFIG_TEST_BLACKHOLE_DEV'}
+        if sub_test in kernel_module_tests.keys():
+            output = utils.system_output('dmesg', retain_output=True)
+            if not output:
+                print("Looks like there's no dmesg output, checking for {}...".format(kernel_module_tests[sub_test]))
+                cmd = "grep ^{} /boot/config-$(uname -r)".format(kernel_module_tests[sub_test])
+                if not utils.system_output(cmd, verbose=False, ignore_status=True):
+                    print("{} not enabled.".format(kernel_module_tests[sub_test]))
+
+        # Old pattern for Xenial
+        pattern = re.compile('selftests: *(?P<case>[\w\-\.]+) \[FAIL\]\n')
+        if re.search(pattern, result):
+            raise error.TestError(test_name + ' failed.')
+        # If the test was not end by previous check, check again with new pattern
+        pattern = re.compile('not ok [\d\.]* selftests: {}: {} # (?!.*SKIP)'.format(category, sub_test))
+        if re.search(pattern, result):
+            raise error.TestError(test_name + ' failed.')
+
+
+# vi:set ts=4 sw=4 expandtab syntax=python: