mirror of
https://github.com/corundum/corundum.git
synced 2025-01-16 08:12:53 +08:00
456 lines
12 KiB
Bash
Executable File
456 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2022, The Regents of the University of California.
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are met:
|
|
#
|
|
# 1. Redistributions of source code must retain the above copyright notice,
|
|
# this list of conditions and the following disclaimer.
|
|
#
|
|
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE REGENTS OF THE UNIVERSITY OF CALIFORNIA ''AS
|
|
# IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OF THE UNIVERSITY OF CALIFORNIA OR
|
|
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
|
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
|
|
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
|
|
# OF SUCH DAMAGE.
|
|
#
|
|
# The views and conclusions contained in the software and documentation are those
|
|
# of the authors and should not be interpreted as representing official policies,
|
|
# either expressed or implied, of The Regents of the University of California.
|
|
|
|
netdev=
|
|
ifaddr=
|
|
netns=
|
|
base_logdir=./logs/
|
|
|
|
max_iperf_count=2
|
|
iperf_repeats=1
|
|
iperf_p=4
|
|
dest_ip=
|
|
iperf_base_port=9000
|
|
mtu=
|
|
|
|
utils_path=../utils/
|
|
mqnic_fw=$utils_path/mqnic-fw
|
|
mqnic_dump=$utils_path/mqnic-dump
|
|
|
|
while getopts i:n:P:c:p:r:-: option; do
|
|
case "${option}" in
|
|
-)
|
|
case "${OPTARG}" in
|
|
ifaddr)
|
|
ifaddr="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
|
|
;;
|
|
ifaddr=*)
|
|
ifaddr=${OPTARG#*=}
|
|
;;
|
|
netns)
|
|
netns="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
|
|
;;
|
|
netns=*)
|
|
netns=${OPTARG#*=}
|
|
;;
|
|
mtu)
|
|
mtu="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
|
|
;;
|
|
mtu=*)
|
|
mtu=${OPTARG#*=}
|
|
;;
|
|
logdir)
|
|
base_logdir="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
|
|
;;
|
|
logdir=*)
|
|
base_logdir=${OPTARG#*=}
|
|
;;
|
|
*)
|
|
if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then
|
|
echo "Unknown option --${OPTARG}" >&2
|
|
fi
|
|
;;
|
|
esac;;
|
|
i) netdev=${OPTARG};;
|
|
n) max_iperf_count=${OPTARG};;
|
|
P) iperf_p=${OPTARG};;
|
|
c) dest_ip=${OPTARG};;
|
|
p) iperf_base_port=${OPTARG};;
|
|
r) iperf_repeats=${OPTARG};;
|
|
esac
|
|
done
|
|
shift $((OPTIND -1))
|
|
|
|
netns_cmd=
|
|
|
|
if [ ! -z "$netdev" ]; then
|
|
if [ ! -z "$ifaddr" ]; then
|
|
if [ -z "$netns" ]; then
|
|
netns=$netdev
|
|
fi
|
|
fi
|
|
else
|
|
echo "Interface name required" >&2
|
|
exit -1
|
|
fi
|
|
|
|
if [ ! -z "$netns" ]; then
|
|
netns_cmd="ip netns exec $netns"
|
|
if [ -f "/var/run/netns/$netns" ]; then
|
|
echo "Network namespace '$netns' already exists"
|
|
else
|
|
echo "Creating network namespace '$netns'"
|
|
ip netns add $netns
|
|
echo "Adding interface '$netdev' to network namespace '$netns'"
|
|
ip link set dev $netdev netns $netns
|
|
$netns_cmd ip link set dev $netdev up
|
|
fi
|
|
fi
|
|
|
|
if [ ! -z "$ifaddr" ]; then
|
|
echo "Adding address '$ifaddr' to '$netdev'"
|
|
$netns_cmd ip addr add $ifaddr dev $netdev
|
|
fi
|
|
|
|
if [ ! -z "$mtu" ]; then
|
|
echo "Changing MTU to $mtu on '$netdev'"
|
|
$netns_cmd ip link set mtu $mtu dev $netdev
|
|
fi
|
|
|
|
function cleanup()
|
|
{
|
|
echo "Cleaning up..."
|
|
|
|
# kill all subprocesses
|
|
trap '' TERM
|
|
pkill -P $$
|
|
|
|
# clean up netns
|
|
if [ ! -z "$netns" ]; then
|
|
if [ -f "/var/run/netns/$netns" -a -z "$(ip netns pids $netns)" ]; then
|
|
echo "Deleting network namespace '$netns'"
|
|
ip netns del $netns
|
|
fi
|
|
fi
|
|
}
|
|
|
|
trap "exit" INT TERM
|
|
trap cleanup EXIT
|
|
|
|
# run tests
|
|
|
|
nic_info=$($netns_cmd $mqnic_dump -d $netdev -v)
|
|
|
|
if [ $? -ne "0" ]; then
|
|
echo "Failed to communicate with device" >2
|
|
exit -1
|
|
fi
|
|
|
|
fpga_id=$(echo "$nic_info" | grep "FPGA ID" | sed -n 's/^.*: //p')
|
|
fpga_part=$(echo "$nic_info" | grep "FPGA part" | sed -n 's/^.*: //p')
|
|
fw_id=$(echo "$nic_info" | grep "FW ID" | sed -n 's/^.*: //p')
|
|
fw_ver=$(echo "$nic_info" | grep "FW version" | sed -n 's/^.*: //p')
|
|
board_id=$(echo "$nic_info" | grep "Board ID" | sed -n 's/^.*: //p')
|
|
board_ver=$(echo "$nic_info" | grep "Board version" | sed -n 's/^.*: //p')
|
|
build_date=$(echo "$nic_info" | grep "Build date" | sed -n 's/^.*: //p')
|
|
git_hash=$(echo "$nic_info" | grep "Git hash" | sed -n 's/^.*: //p')
|
|
release_info=$(echo "$nic_info" | grep "Release info" | sed -n 's/^.*: //p')
|
|
|
|
logname="$(date +%Y%m%dT%H%M%S)_${board_id}_fw${fw_ver}_${git_hash}"
|
|
|
|
logdir=$base_logdir/$logname
|
|
|
|
mkdir -p $logdir
|
|
|
|
echo "$nic_info" > $logdir/mqnic_dump_pre.log
|
|
|
|
{
|
|
|
|
tests=()
|
|
declare -A test_desc
|
|
declare -A test_status
|
|
declare -A test_status_desc
|
|
|
|
link_ok=0
|
|
|
|
echo "Firmware information"
|
|
echo "FPGA ID: $fpga_id"
|
|
echo "FPGA part: $fpga_part"
|
|
echo "FW ID: $fw_id"
|
|
echo "FW version: $fw_ver"
|
|
echo "Board ID: $board_id"
|
|
echo "Board version: $board_ver"
|
|
echo "Build date: $build_date"
|
|
echo "Git hash: $git_hash"
|
|
echo "Release info: $release_info"
|
|
|
|
echo "PCIe information from lspci"
|
|
pci_id=$(basename $($netns_cmd readlink /sys/class/net/$netdev/device))
|
|
lspci -vvv -s $pci_id | tee $logdir/lspci.log 2>&1
|
|
|
|
echo "CPU information from lscpu"
|
|
lscpu | tee $logdir/lscpu.log 2>&1
|
|
cpu_model=$(cat /proc/cpuinfo | grep name | head -1 | cut -d: -f 2 | xargs)
|
|
|
|
echo "Link information from iproute2"
|
|
$netns_cmd ip link show dev $netdev | tee $logdir/iproute2_link.log 2>&1
|
|
|
|
echo "Network device information from ethtool"
|
|
$netns_cmd ethtool $netdev | tee $logdir/ethtool.log 2>&1
|
|
|
|
echo "Network device features from ethtool"
|
|
$netns_cmd ethtool -k $netdev | tee $logdir/ethtool_k.log 2>&1
|
|
|
|
|
|
tests+=("macaddr")
|
|
test_desc[macaddr]="Default MAC address"
|
|
test_status[macaddr]="PASS"
|
|
test_status_desc[macaddr]="OK"
|
|
echo "Current test: " ${test_desc[macaddr]}
|
|
|
|
mac=$($netns_cmd ip link show dev $netdev | grep link/ether | tr -s ' ' | cut -d' ' -f 3)
|
|
|
|
echo "Assigned MAC: $mac"
|
|
|
|
b=$(echo $mac | cut -d : -f 1)
|
|
|
|
if [ $((16#$b & 1)) -ne 0 ]; then
|
|
echo "Invalid assigned MAC (broadcast bit set)"
|
|
test_status[macaddr]="FAIL"
|
|
test_status_desc[macaddr]="Invalid MAC"
|
|
fi
|
|
|
|
if [ $((16#$b & 2)) -ne 0 ]; then
|
|
echo "Locally assigned MAC; persistent MAC addressing not working or EEPROM is blank"
|
|
test_status[macaddr]="FAIL"
|
|
test_status_desc[macaddr]="Local MAC"
|
|
fi
|
|
|
|
if [ "${test_status[macaddr]}" = "PASS" ]; then
|
|
echo "MAC address OK"
|
|
fi
|
|
|
|
|
|
tests+=("mod_i2c")
|
|
test_desc[mod_i2c]="Module communication"
|
|
test_status[mod_i2c]="PASS"
|
|
test_status_desc[mod_i2c]="OK"
|
|
echo "Current test: ${test_desc[mod_i2c]}"
|
|
|
|
$netns_cmd ethtool -m $netdev | tee $logdir/ethtool_m.log 2>&1
|
|
|
|
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
|
echo "Failed to read module EEPROM"
|
|
test_status[mod_i2c]="FAIL"
|
|
test_status_desc[mod_i2c]="Failed to read EEPROM"
|
|
else
|
|
echo "Successfully read module EEPROM"
|
|
fi
|
|
|
|
|
|
tests+=("ping")
|
|
test_desc[ping]="Ping"
|
|
test_status[ping]="PASS"
|
|
test_status_desc[ping]="OK"
|
|
echo "Current test: ${test_desc[ping]}"
|
|
|
|
$netns_cmd ping -c 4 -w 10 $dest_ip | tee $logdir/ping.log 2>&1
|
|
|
|
if [ ${PIPESTATUS[0]} -ne 0 ]; then
|
|
echo "Cannot communicate with test host"
|
|
test_status[ping]="FAIL"
|
|
test_status_desc[ping]="No response"
|
|
link_ok=0
|
|
else
|
|
echo "Communication with test host OK"
|
|
link_ok=1
|
|
fi
|
|
|
|
|
|
if [ $link_ok -ne 0 ]; then
|
|
echo "Link partner NIC MAC address from iproute2"
|
|
$netns_cmd ip neigh show $dest_ip | tee $logdir/iproute2_neigh.log 2>&1
|
|
remote_mac=$($netns_cmd ip neigh show $dest_ip | cut -d' ' -f 5)
|
|
fi
|
|
|
|
|
|
ptp4l_pid=0
|
|
|
|
if [ $link_ok -ne 0 ]; then
|
|
echo "Start ptp4l"
|
|
|
|
$netns_cmd ptp4l -i $netdev --slaveOnly=1 --tx_timestamp_timeout=10 -m > $logdir/ptp4l.log 2>&1 &
|
|
ptp4l_pid=$!
|
|
fi
|
|
|
|
|
|
tests+=("ptp_sync1")
|
|
test_desc[ptp_sync1]="PTP sync at idle"
|
|
test_status[ptp_sync1]="PASS"
|
|
test_status_desc[ptp_sync1]="OK"
|
|
echo "Current test: ${test_desc[ptp_sync1]}"
|
|
|
|
|
|
if [ $link_ok -ne 0 ]; then
|
|
|
|
echo "Wait for PTP to sync"
|
|
tail -f $logdir/ptp4l.log &
|
|
tail_pid=$!
|
|
|
|
sleep 60
|
|
|
|
kill $tail_pid
|
|
|
|
ptp_status=$(cat $logdir/ptp4l.log | grep freq | grep rms | tail -1)
|
|
|
|
echo $ptp_status
|
|
|
|
ptp_offset_rms=$(echo "$ptp_status" | tr -s ' ' | cut -d ' ' -f 3)
|
|
|
|
if [ ! -z "$ptp_offset_rms" ]; then
|
|
echo "RMS offset: $ptp_offset_rms"
|
|
|
|
if [ "$ptp_offset_rms" -lt 100 ]; then
|
|
echo "PTP sync OK"
|
|
else
|
|
echo "PTP sync not sufficiently accurate"
|
|
echo "Note: this may caused by other components in the network not properly supporting PTP"
|
|
test_status[ptp_sync1]="FAIL"
|
|
test_status_desc[ptp_sync1]="Inaccurate sync"
|
|
fi
|
|
else
|
|
echo "PTP failed to sync"
|
|
test_status[ptp_sync1]="FAIL"
|
|
test_status_desc[ptp_sync1]="Did not sync"
|
|
fi
|
|
|
|
else
|
|
echo "Skipping test due to link issue"
|
|
test_status[ptp_sync1]="SKIP"
|
|
test_status_desc[ptp_sync1]="No link"
|
|
fi
|
|
|
|
|
|
tests+=("bandwidth")
|
|
test_desc[bandwidth]="Bandwidth"
|
|
test_status[bandwidth]="PASS"
|
|
test_status_desc[bandwidth]="OK"
|
|
echo "Current test: ${test_desc[bandwidth]}"
|
|
|
|
|
|
if [ $link_ok -ne 0 ]; then
|
|
|
|
$netns_cmd ./iperf_benchmark.sh --logdir "$logdir/iperf" -c $dest_ip -P $iperf_p -n $max_iperf_count -r $iperf_repeats | tee $logdir/iperf_benchmark.log 2>&1
|
|
|
|
else
|
|
echo "Skipping test due to link issue"
|
|
test_status[bandwidth]="SKIP"
|
|
test_status_desc[bandwidth]="No link"
|
|
fi
|
|
|
|
|
|
if [ $ptp4l_pid -ne 0 ]; then
|
|
echo "Stop ptp4l"
|
|
kill $ptp4l_pid
|
|
fi
|
|
|
|
|
|
tests+=("ptp_sync2")
|
|
test_desc[ptp_sync2]="PTP sync under load"
|
|
test_status[ptp_sync2]="PASS"
|
|
test_status_desc[ptp_sync2]="OK"
|
|
echo "Current test: ${test_desc[ptp_sync2]}"
|
|
|
|
|
|
if [ $link_ok -ne 0 ]; then
|
|
|
|
ptp_status=$(cat $logdir/ptp4l.log | grep freq | grep rms | tail -1)
|
|
|
|
echo $ptp_status
|
|
|
|
ptp_offset_rms=$(echo "$ptp_status" | tr -s ' ' | cut -d ' ' -f 3)
|
|
|
|
if [ ! -z "$ptp_offset_rms" ]; then
|
|
echo "RMS offset: $ptp_offset_rms"
|
|
|
|
if [ "$ptp_offset_rms" -lt 100 ]; then
|
|
echo "PTP sync OK"
|
|
else
|
|
echo "PTP sync not sufficiently accurate"
|
|
echo "Note: this may caused by other components in the network not properly supporting PTP"
|
|
test_status[ptp_sync2]="FAIL"
|
|
test_status_desc[ptp_sync2]="Inaccurate sync"
|
|
fi
|
|
else
|
|
echo "PTP failed to sync"
|
|
test_status[ptp_sync2]="FAIL"
|
|
test_status_desc[ptp_sync2]="Did not sync"
|
|
fi
|
|
|
|
else
|
|
echo "Skipping test due to link issue"
|
|
test_status[ptp_sync2]="SKIP"
|
|
test_status_desc[ptp_sync2]="No link"
|
|
fi
|
|
|
|
echo
|
|
|
|
{
|
|
|
|
echo "Test summary"
|
|
echo
|
|
echo "Date: $(date)"
|
|
echo
|
|
echo "Design information"
|
|
echo
|
|
echo " FPGA ID: $fpga_id"
|
|
echo " FPGA part: $fpga_part"
|
|
echo " FW ID: $fw_id"
|
|
echo " FW version: $fw_ver"
|
|
echo " Board ID: $board_id"
|
|
echo " Board version: $board_ver"
|
|
echo " Build date: $build_date"
|
|
echo " Git hash: $git_hash"
|
|
echo " Release info: $release_info"
|
|
echo
|
|
echo " PCI ID: $pci_id"
|
|
echo " MAC address: $mac"
|
|
echo
|
|
echo "Test setup"
|
|
echo
|
|
echo " CPU: $cpu_model"
|
|
echo " Remote MAC address: $remote_mac"
|
|
echo
|
|
echo "Test results"
|
|
echo
|
|
|
|
overall="PASS"
|
|
|
|
for test in "${tests[@]}"; do
|
|
printf " %-30s: %-4s (%s)\n" "${test_desc[$test]}" "${test_status[$test]}" "${test_status_desc[$test]}"
|
|
|
|
if [ "${test_status[$test]}" = "FAIL" ]; then
|
|
overall="FAIL"
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "Overall result: $overall"
|
|
echo
|
|
|
|
} | tee $logdir/summary.log 2>&1
|
|
|
|
echo "Done"
|
|
|
|
} | tee $logdir/test.log 2>&1
|
|
|
|
$netns_cmd $mqnic_dump -d $netdev -v > $logdir/mqnic_dump_post.log
|