Merge ~liaou3/checkbox-iiotg/+git/checkbox-provider-intliotg:refactor_tsn into ~checkbox-dev/checkbox-iiotg/+git/checkbox-provider-intliotg:master

[Vincent Liao]

Modify: make tsn-ieee802.1as-2011 tests automated

This MR is to do the automation of tsn-ieee802.1as-2011.

submission: https://certification.canonical.com/hardware/202306-31658/submission/341151/

[PeiYao Chang (baconyao)]

Please see my inline comments

review: Needs Fixing

[Vincent Liao (liaou3)]

> Please see my inline comments
Fixed. Thanks!

[PeiYao Chang (baconyao)]

LGTM

review: Approve

[Rick Wu (rickwu4444)]

@Vincent,
Please see my inline comment.

review: Needs Information

[Vincent Liao (liaou3)]

> @Vincent,
> Please see my inline comment.
I forgot to add them into environ field. Thanks for the reminder.

[Rick Wu (rickwu4444)]

LGTM

review: Approve

diff --git a/bin/tsn_phc2sys.sh b/bin/tsn_phc2sys.sh
new file mode 100755
index 0000000..e848864
--- /dev/null
+++ b/bin/tsn_phc2sys.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -e
+
+timeout=$1
+mode=$2
+eth_interface=$3
+server_user=$4
+server_ip=$5
+server_pwd=$6
+server_eth_interface=$7
+
+# Start time sync as ${mode}
+tsn_time_sync.sh "$timeout" "$mode" "$eth_interface" "$server_user" "$server_ip" "$server_pwd" "$server_eth_interface" > /dev/null &
+
+# start sync time between physical hardware clock and system clock
+out=$(mktemp)
+# Use phc2sys here instead of linuxptp-rt.phc2sys is because there are some issue with the second one.
+timeout "$timeout" phc2sys -s "$eth_interface" -O 0 -c CLOCK_REALTIME --step_threshold=1 --transportSpecific=1 -w -m | tee "$out"
+echo ""
+
+# Checking the last 10 seconds is because the offset value is not stable enough in the beginning 
+lines=$(tail -n 10 "$out")
+exit_code=0
+while read -r line
+do
+    offset=$(echo "$line" | grep -o 'phc offset.*freq' | awk '{print $3}')
+    state=$(echo "$line" | grep -o 'phc offset.*freq' | awk '{print $4}')
+    delay=$(echo "$line" | awk '{print $10}')
+    if [ -n "$offset_failed" ] && [ "$offset" -gt 100 ] || [ "$offset" -lt -100 ]; then
+        echo "FAIL: phc offset is not within -100 to 100 for the last 10 seconds"
+        echo "phc offset is the offset between system clock and physical hardware clock"
+        echo ""
+        offset_failed=1
+        exit_code=1
+    fi
+    if [ -n "$state_failed" ] && [ "$state" != "s2" ]; then
+        echo "FAIL: state is not equal to s2 for the last 10 seconds"
+        echo "s0: unsynced"
+        echo "s1: syncing"
+        echo "s2: synced"
+        echo ""
+        state_failed=1
+        exit_code=1
+    fi
+    if [ -n "$delay_failed" ] && [ "$delay" -ne 0 ]; then
+        echo "FAIL: path delay is not equal to 0 for the last 10 seconds"
+        echo "path delay should be 0 if using hardware cross timestamping"
+        echo ""
+        delay_failed=1
+        exit_code=1
+    fi
+done <<< "$lines"
+
+if [ $exit_code -eq 0 ]; then
+    echo "PASS: phc offset is within -100 to 100 for the last 10 seconds"
+    echo "PASS: state is equal to s2 (synced)"
+    echo "PASS: path delay is equal to 0"
+fi
diff --git a/bin/tsn_time_sync.sh b/bin/tsn_time_sync.sh
index c8e3ce9..9b18793 100755
--- a/bin/tsn_time_sync.sh
+++ b/bin/tsn_time_sync.sh
@@ -4,23 +4,25 @@ set -e
 
 timeout=$1
 mode=$2
-config_path=$3
-eth_interface=$4
-server_user=$5
-server_ip=$6
-server_pwd=$7
-server_eth_interface=$8
+eth_interface=$3
+server_user=$4
+server_ip=$5
+server_pwd=$6
+server_eth_interface=$7
+
+master_config="/snap/linuxptp-rt/current/etc/automotive-master.cfg"
+client_config="/snap/linuxptp-rt/current/etc/automotive-slave.cfg"
 
 # This will clear any queue discipline configuration on the devices. If there was no configuration, you will get an error, but you can ignore it.
 sshpass -p "$server_pwd" ssh -o StrictHostKeyChecking=no "$server_user"@"$server_ip" "echo ${server_pwd} | sudo -S tc qdisc del dev $server_eth_interface root" || true
 tc qdisc del dev "$eth_interface" root || true
 
 if [ "$mode" == "client" ]; then
-    sshpass -p "$server_pwd" ssh -o StrictHostKeyChecking=no "$server_user"@"$server_ip" "echo ${server_pwd} | sudo -S timeout $timeout ptp4l -P2Hi $server_eth_interface -f $config_path --step_threshold=1 --first_step_threshold=0.0" &
-    timeout "$timeout" ptp4l -P2Hi "$eth_interface" -f "$config_path" --step_threshold=1 -m -s --first_step_threshold=0.0
+    sshpass -p "$server_pwd" ssh -o StrictHostKeyChecking=no "$server_user"@"$server_ip" "echo ${server_pwd} | sudo -S timeout $timeout linuxptp-rt.ptp4l -i $server_eth_interface -f $master_config --step_threshold=1" &
+    timeout "$timeout" linuxptp-rt.ptp4l -i "$eth_interface" -f $client_config --step_threshold=1 -m -s --first_step_threshold=0.0
 elif [ "$mode" == "master" ]; then
-    timeout "$timeout" ptp4l -P2Hi "$eth_interface" -f "$config_path" --step_threshold=1 --first_step_threshold=0.0 &
-    sshpass -p "$server_pwd" ssh -o StrictHostKeyChecking=no "$server_user"@"$server_ip" "echo ${server_pwd} | sudo -S timeout $timeout ptp4l -m -s -P2Hi $server_eth_interface -f $config_path --step_threshold=1 --first_step_threshold=0.0"
+    timeout "$timeout" linuxptp-rt.ptp4l -i "$eth_interface" -f "$master_config" --step_threshold=1 &
+    sshpass -p "$server_pwd" ssh -o StrictHostKeyChecking=no "$server_user"@"$server_ip" "echo ${server_pwd} | sudo -S timeout $timeout linuxptp-rt.ptp4l -m -s -i $server_eth_interface -f $client_config --step_threshold=1"
 else
     echo "Wrong argument for $mode. It should be either client or master"
     exit 1
diff --git a/units/tsn/jobs.pxu b/units/tsn/jobs.pxu
index 3bd09c6..de2cbae 100644
--- a/units/tsn/jobs.pxu
+++ b/units/tsn/jobs.pxu
@@ -144,120 +144,125 @@ command:
 unit: template
 template-resource: tsn/tsn-devices
 template-unit: job
-id: tsn/ieee802.1as-2011/time-sync-between-two-system-as-client-{interface}
+template-engine: jinja2
+id: tsn/ieee802.1as-2011/time-sync-between-two-system-as-client-{{ interface }}
 category_id: tsn
 _summary:
-    Check establish time-sync with another machine as client via {interface}
+    Check establish time-sync with another machine as client via {{ interface }}
 _purpose:
-    Use {interface} on the SUT to achieve time sync with the second machine as client
+    Use {{ interface }} on the SUT to achieve time sync with the second machine as client
 depends:
     tsn/tsn-detected
     tsn/testing-snap-install-check
-plugin: user-interact-verify
+plugin: shell
 user: root
 environ:
     TSN_SERVER_IP
     TSN_SERVER_USER
     TSN_SERVER_PWD
     TSN_SERVER_INTERFACE
-    GPTP_CONFIG_PATH
+    TSN_MAX_MASTER_OFFSET
+    TSN_SYNC_TIME
 command:
-    tsn_time_sync.sh 60 client "${{GPTP_CONFIG_PATH:-/usr/share/gPTP.cfg}}" {interface} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE"
-_verification:
-    Is the `rms` value within 100ns?
+    set -e
+    out=$(mktemp)
+    tsn_time_sync.sh ${TSN_SYNC_TIME:-60} client {{ interface }} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE" | tee "$out"
+    echo ""
+    lines=$(tail -n 10 "$out")
+    while read -r line
+    do
+        offset=$(echo "$line" | grep -o 'master offset\s*-\?[0-9]\+ s' | awk '{print $3}')
+        if [ "$offset" -gt "${TSN_MAX_MASTER_OFFSET:-200}" ] || [ "$offset" -lt -"${TSN_MAX_MASTER_OFFSET:-200}" ]; then
+            echo "FAIL: master offset is not within -"${TSN_MAX_MASTER_OFFSET:-200}" to "${TSN_MAX_MASTER_OFFSET:-200}" for the last 10 seconds."
+            exit 1
+        fi
+    done <<< "$lines"
+    echo "PASS: master offset is within -"${TSN_MAX_MASTER_OFFSET:-200}" to "${TSN_MAX_MASTER_OFFSET:-200}" for the last 10 seconds."
 
 unit: template
 template-resource: tsn/tsn-devices
 template-unit: job
-id: tsn/ieee802.1as-2011/time-sync-between-two-system-as-master-{interface}
+template-engine: jinja2
+id: tsn/ieee802.1as-2011/time-sync-between-two-system-as-master-{{ interface }}
 category_id: tsn
 _summary:
-    Check establish time-sync with another machine as master via {interface}
+    Check establish time-sync with another machine as master via {{ interface }}
 _purpose:
-    Use {interface} on the SUT to achieve time sync with the second machine as master
+    Use {{ interface }} on the SUT to achieve time sync with the second machine as master
 depends:
     tsn/tsn-detected
-    tsn/testing-snap-install-check
-plugin: user-interact-verify
+plugin: shell
 user: root
 environ:
     TSN_SERVER_IP
     TSN_SERVER_USER
     TSN_SERVER_PWD
     TSN_SERVER_INTERFACE
-    GPTP_CONFIG_PATH
+    TSN_MAX_MASTER_OFFSET
+    TSN_SYNC_TIME
 command:
-    tsn_time_sync.sh 60 master "${{GPTP_CONFIG_PATH:-/usr/share/gPTP.cfg}}" {interface} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE"
-_verification:
-    Is the `rms` value within 100ns?
+    set -e
+    out=$(mktemp)
+    tsn_time_sync.sh ${TSN_SYNC_TIME:-60} master {{ interface }} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE" | tee "$out"
+    echo ""
+    lines=$(tail -n 10 "$out")
+    while read -r line
+    do
+        offset=$(echo "$line" | grep -o 'master offset\s*-\?[0-9]\+ s' | awk '{print $3}')
+        if [ "$offset" -gt "${TSN_MAX_MASTER_OFFSET:-200}" ] || [ "$offset" -lt -"${TSN_MAX_MASTER_OFFSET:-200}" ]; then
+            echo "FAIL: master offset is not within -"${TSN_MAX_MASTER_OFFSET:-200}" to "${TSN_MAX_MASTER_OFFSET:-200}" for the last 10 seconds."
+            exit 1
+        fi
+    done <<< "$lines"
+    echo "PASS: master offset is within -"${TSN_MAX_MASTER_OFFSET:-200}" to "${TSN_MAX_MASTER_OFFSET:-200}" for the last 10 seconds."
 
 unit: template
 template-resource: tsn/tsn-devices
 template-unit: job
-id: tsn/ieee802.1as-2011/time-sync-system-clock-with-PHC-as-client-{interface}
+template-engine: jinja2
+id: tsn/ieee802.1as-2011/time-sync-system-clock-with-PHC-as-client-{{ interface }}
 category_id: tsn
 _summary:
-    Check system time sync with PTP hardware clock as client
+    System time sync with PTP hardware clock as client
 depends:
     tsn/tsn-detected
-    tsn/testing-snap-install-check
-    tsn/ieee802.1as-2011/time-sync-between-two-system-as-client-{interface}
-plugin: user-interact-verify
+    tsn/ieee802.1as-2011/time-sync-between-two-system-as-client-{{ interface }}
+plugin: shell
 user: root
 _purpose:
-    To check if the system can sync time between system clock and PTP hardware clock as client
+    To check if the system can sync time between system clock and PTP hardware clock as client via {{ interface }}
 environ:
     TSN_SERVER_IP
     TSN_SERVER_USER
     TSN_SERVER_PWD
     TSN_SERVER_INTERFACE
-    GPTP_CONFIG_PATH
+    TSN_SYNC_TIME
 command:
-    tsn_time_sync.sh 60 client "${{GPTP_CONFIG_PATH:-/usr/share/gPTP.cfg}}" {interface} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE" > /dev/null &
-    pmc -u -b 0 -t 1 "SET GRANDMASTER_SETTINGS_NP clockClass 248\
-    clockAccuracy 0xfe offsetScaledLogVariance 0xffff currentUtcOffset 37\
-    leap61 0 leap59 0 currentUtcOffsetValid 1 ptpTimescale 1 timeTraceable 1\
-    frequencyTraceable 0 timeSource 0xa0"
-    timeout 60 phc2sys -s {interface} -O 0 -c CLOCK_REALTIME --step_threshold=1 \
-    --transportSpecific=1 -w -m --first_step_threshold=0.0
-_verification:
-    Is the `phc offset` value within `100ns`?
-    Is the `State` value equal to `S2`?
-    Is the `Path Delay` equal to `0`
+    tsn_phc2sys.sh ${TSN_SYNC_TIME:-60} client {{ interface }} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE"
 
 unit: template
 template-resource: tsn/tsn-devices
 template-unit: job
-id: tsn/ieee802.1as-2011/time-sync-system-clock-with-PHC-as-master-{interface}
+template-engine: jinja2
+id: tsn/ieee802.1as-2011/time-sync-system-clock-with-PHC-as-master-{{ interface }}
 category_id: tsn
 _summary:
-    Check system time sync with PTP hardware clock as client
+    System time sync with PTP hardware clock as master
 depends:
     tsn/tsn-detected
-    tsn/testing-snap-install-check
-    tsn/ieee802.1as-2011/time-sync-between-two-system-as-master-{interface}
-plugin: user-interact-verify
+    tsn/ieee802.1as-2011/time-sync-between-two-system-as-master-{{ interface }}
+plugin: shell
 user: root
 _purpose:
-    To check if the system can sync time between system clock and PTP hardware clock as master
+    To check if the system can sync time between system clock and PTP hardware clock as master via {{ interface }}
 environ:
     TSN_SERVER_IP
     TSN_SERVER_USER
     TSN_SERVER_PWD
     TSN_SERVER_INTERFACE
-    GPTP_CONFIG_PATH
+    TSN_SYNC_TIME
 command:
-    tsn_time_sync.sh 60 master "${{GPTP_CONFIG_PATH:-/usr/share/gPTP.cfg}}" {interface} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE" > /dev/null &
-    pmc -u -b 0 -t 1 "SET GRANDMASTER_SETTINGS_NP clockClass 248\
-    clockAccuracy 0xfe offsetScaledLogVariance 0xffff currentUtcOffset 37\
-    leap61 0 leap59 0 currentUtcOffsetValid 1 ptpTimescale 1 timeTraceable 1\
-    frequencyTraceable 0 timeSource 0xa0"
-    timeout 60 phc2sys -s {interface} -O 0 -c CLOCK_REALTIME --step_threshold=1 \
-    --transportSpecific=1 -w -m --first_step_threshold=0.0
-_verification:
-    Is the `phc offset` value within `100ns`?
-    Is the `State` value equal to `S2`?
-    Is the `Path Delay` equal to `0`
+    tsn_phc2sys.sh ${TSN_SYNC_TIME:-60} master {{ interface }} "$TSN_SERVER_USER" "$TSN_SERVER_IP" "$TSN_SERVER_PWD" "$TSN_SERVER_INTERFACE"
 
 unit: template
 template-resource: tsn/tsn-devices
diff --git a/units/tsn/test-plan.pxu b/units/tsn/test-plan.pxu
index fd5653c..025bb18 100644
--- a/units/tsn/test-plan.pxu
+++ b/units/tsn/test-plan.pxu
@@ -18,7 +18,6 @@ include:
     tsn/test-env-setup-for-.*
     tsn/testing-snap-install-check
 nested_part:
-    tsn-ieee802.1as-2011
     tsn-time-based-shaping
     tsn-ieee802.1qbv
 
@@ -32,6 +31,7 @@ include:
     tsn/test-env-setup-for-.*
     tsn/testing-snap-install-check
 nested_part:
+    tsn-ieee802.1as-2011
     tsn-ieee802.1qav
     tsn-ieee802.1qbu