Merge ~kevinbecker/+git/autotest-client-tests:kevinbecker/rt-thresholds into ~canonical-kernel-team/+git/autotest-client-tests:master

[Kevin Becker]

UBUNTU: SAUCE: Update max latency handling of rt tests to better handle ivysaur

The original limits were set for starlow and were probably set a little too tightly. Relax limits since we only want to fail on a clearly too high max latency. This commit also allows per host latency limits since ivysaur has more latency than starlow.

Signed-off-by: Kevin Becker <email address hidden>

[Po-Hsu Lin (cypressyew)]

Hi Kevin,
overall this is looking good.
But there is one line need to be fixed, and I have some suggestions to add '-b' flag into args. As it's indeed an arg that we need to pass into the program. In this way I think it's more readable.

Please find inline comments.
Thanks

review: Needs Fixing

[Kevin Becker (kevinbecker)]

> Hi Kevin,
> overall this is looking good.
> But there is one line need to be fixed, and I have some suggestions to add
> '-b' flag into args. As it's indeed an arg that we need to pass into the
> program. In this way I think it's more readable.
>
> Please find inline comments.
> Thanks

Thanks Sam! Good catch on the missing space. I've incorporated your changes and I will retest on starlow later today.

[Po-Hsu Lin (cypressyew)]

+1.
Let's move on to get this landed. Please feel free to open a new MP for more adjustments.
Applied with comment break in multiple lines and pushed.

[Po-Hsu Lin (cypressyew)]

review: Approve

[Kevin Becker (kevinbecker)]

Thanks!

diff --git a/rt_tests_cyclictest/rt_tests_cyclictest.py b/rt_tests_cyclictest/rt_tests_cyclictest.py
index ac8d2ac..4cd1cb1 100644
--- a/rt_tests_cyclictest/rt_tests_cyclictest.py
+++ b/rt_tests_cyclictest/rt_tests_cyclictest.py
@@ -12,6 +12,7 @@ class rt_tests_cyclictest(test.test):
     def initialize(self):
         self.flavour = re.split('-\d*-', platform.uname()[2])[-1]
         self.arch = platform.processor()
+        self.hostname = os.uname()[1]
 
     def install_required_pkgs(self):
         try:
@@ -65,16 +66,23 @@ class rt_tests_cyclictest(test.test):
     #
     #    Runs cyclictest with 10 threads, for 100000 loops, priority set to 
     #    80 and an interval of 200us. It will fail if the max latency goes 
-    #    over 100us.
+    #    over a specified latency.
     #
-    def run_once(self, test_name, args='-t 10 -m -l 100000 -p 80 -b 100 -i 200 -d 0', exit_on_error=True):
+    def run_once(self, test_name, args='-t 10 -m -l 100000 -p 80 -i 200 -d 0', exit_on_error=True):
         if test_name == 'setup':
             return
+        
+        latency_limit = 1000
+        if self.hostname == 'starlow':
+            latency_limit = 200
+        elif self.hostname == 'ivysaur':
+            latency_limit = 700
 
+        args += ' -b ' + str(latency_limit)
         self.results = utils.system_output(self.srcdir + '/rt-tests/cyclictest ' + args, retain_output=True)
 
         if "Break" == self.results.splitlines()[-1].split()[1]:
-            raise error.TestError('FAIL: Max latency over 100us.')
+            raise error.TestError('FAIL: Max latency over ' + str(latency_limit) + 'us.')
 
         return
 
diff --git a/rt_tests_pmqtest/rt_tests_pmqtest.py b/rt_tests_pmqtest/rt_tests_pmqtest.py
index d9e3fda..a55565a 100644
--- a/rt_tests_pmqtest/rt_tests_pmqtest.py
+++ b/rt_tests_pmqtest/rt_tests_pmqtest.py
@@ -12,6 +12,7 @@ class rt_tests_pmqtest(test.test):
     def initialize(self):
         self.flavour = re.split('-\d*-', platform.uname()[2])[-1]
         self.arch = platform.processor()
+        self.hostname = os.uname()[1]
 
     def install_required_pkgs(self):
         try:
@@ -63,12 +64,19 @@ class rt_tests_pmqtest(test.test):
     #
     #    Driven by the control file for each individual test.
     #
-    #    Runs pmqtest for 60 seconds. This will fail if the max latency is over 100us.
+    #    Runs pmqtest for 60 seconds. This will fail if the max latency is over a specified limit.
     #
-    def run_once(self, test_name, args='-Sp80 -i100 -d0 -b100 -q -D60', exit_on_error=True):
+    def run_once(self, test_name, args='-Sp80 -i100 -d0 -q -D60', exit_on_error=True):
         if test_name == 'setup':
             return
+        
+        latency_limit = 1000
+        if self.hostname == 'starlow':
+            latency_limit = 200
+        elif self.hostname == 'ivysaur':
+            latency_limit = 700
 
+        args += ' -b' + str(latency_limit)
         self.results = utils.system_output(self.srcdir + '/rt-tests/pmqtest ' + args, retain_output=True)
 
         lines = self.results.split('\n')
@@ -78,8 +86,8 @@ class rt_tests_pmqtest(test.test):
             if 'Max' in line:
                 max_latencies.append(int(line.split()[-1]))
 
-        # Check if any max value is over 100
-        if any(value > 100 for value in max_latencies):
-            raise error.TestError('FAIL: Max latency over 100us.')
+        # Check if any max value is over latency limit
+        if any(value > latency_limit for value in max_latencies):
+            raise error.TestError('FAIL: Max latency over ' + str(latency_limit) + 'us.')
 
         return
diff --git a/rt_tests_ptsematest/rt_tests_ptsematest.py b/rt_tests_ptsematest/rt_tests_ptsematest.py
index 4bd6456..aa29b96 100644
--- a/rt_tests_ptsematest/rt_tests_ptsematest.py
+++ b/rt_tests_ptsematest/rt_tests_ptsematest.py
@@ -12,6 +12,7 @@ class rt_tests_ptsematest(test.test):
     def initialize(self):
         self.flavour = re.split('-\d*-', platform.uname()[2])[-1]
         self.arch = platform.processor()
+        self.hostname = os.uname()[1]
 
     def install_required_pkgs(self):
         try:
@@ -64,11 +65,17 @@ class rt_tests_ptsematest(test.test):
     #    Driven by the control file for each individual test.
     #
     #    Runs ptsematest with one thread per processor, for 100000 loops, and 
-    #    priority set to 80. It will fail if the max latency goes over 100us.
+    #    priority set to 80. It will fail if the max latency goes over a specified limit.
     #
     def run_once(self, test_name, args='-l 100000 -p 80 -S -q', exit_on_error=True):
         if test_name == 'setup':
             return
+        
+        latency_limit = 1000
+        if self.hostname == 'starlow':
+            latency_limit = 200
+        elif self.hostname == 'ivysaur':
+            latency_limit = 700
 
         self.results = utils.system_output(self.srcdir + '/rt-tests/ptsematest ' + args, retain_output=True)
 
@@ -88,7 +95,7 @@ class rt_tests_ptsematest(test.test):
         highest_max = max(max_values)
         print("Highest Max Latency:", highest_max)
 
-        if highest_max > 100:
-            raise error.TestError('FAIL: Max latency over 100us.')
+        if highest_max > latency_limit:
+            raise error.TestError('FAIL: Max latency over ' + str(latency_limit) + 'us.')
 
         return
diff --git a/rteval/rteval.py b/rteval/rteval.py
index 15c9b2a..821c763 100644
--- a/rteval/rteval.py
+++ b/rteval/rteval.py
@@ -14,6 +14,7 @@ class rteval(test.test):
     def initialize(self):
         self.flavour = re.split('-\d*-', platform.uname()[2])[-1]
         self.arch = platform.processor()
+        self.hostname = os.uname()[1]
 
     def install_required_pkgs(self):
         try:
@@ -100,7 +101,7 @@ class rteval(test.test):
     #
     #    Driven by the control file for each individual test.
     #
-    #    Runs rteval. Test passes if max latency is not over 200us.
+    #    Runs rteval. Test passes if max latency is not over specified limit.
     #
     def run_once(self, test_name, args='', exit_on_error=True):
         if test_name == 'setup':
@@ -137,7 +138,13 @@ class rteval(test.test):
         latency = maximum_tag.text
         print("Maximum latency: "+latency+"us")
 
-        if int(latency) > 200:
-            raise error.TestError('FAIL: Max latency too high.')
+        latency_limit = 1000
+        if self.hostname == 'starlow':
+            latency_limit = 200
+        elif self.hostname == 'ivysaur':
+            latency_limit = 800
+
+        if int(latency) > latency_limit:
+            raise error.TestError('FAIL: Max latency over ' + str(latency_limit) + 'us.')
 
         return