From 36e202352b8a98fdd0b324a570bcf4bbc7b20208 Mon Sep 17 00:00:00 2001 From: Alex Forencich Date: Wed, 18 May 2022 20:04:45 -0700 Subject: [PATCH] Add NIC test script Signed-off-by: Alex Forencich --- scripts/test_mqnic.sh | 430 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100755 scripts/test_mqnic.sh diff --git a/scripts/test_mqnic.sh b/scripts/test_mqnic.sh new file mode 100755 index 000000000..c7541da8b --- /dev/null +++ b/scripts/test_mqnic.sh @@ -0,0 +1,430 @@ +#!/bin/bash + +# Copyright 2022, The Regents of the University of California. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS OF THE UNIVERSITY OF CALIFORNIA ''AS +# IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS OF THE UNIVERSITY OF CALIFORNIA OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT +# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY +# OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of The Regents of the University of California. + +netdev= +ifaddr= +netns= +base_logdir=./logs/ + +max_iperf_count=2 +iperf_repeats=1 +iperf_p=4 +dest_ip= +iperf_base_port=9000 + +utils_path=../utils/ +mqnic_fw=$utils_path/mqnic-fw +mqnic_dump=$utils_path/mqnic-dump + +while getopts i:n:P:c:p:r:-: option; do + case "${option}" in + -) + case "${OPTARG}" in + ifaddr) + ifaddr="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + ifaddr=*) + ifaddr=${OPTARG#*=} + ;; + netns) + netns="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + netns=*) + netns=${OPTARG#*=} + ;; + logdir) + base_logdir="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + logdir=*) + base_logdir=${OPTARG#*=} + ;; + *) + if [ "$OPTERR" = 1 ] && [ "${optspec:0:1}" != ":" ]; then + echo "Unknown option --${OPTARG}" >&2 + fi + ;; + esac;; + i) netdev=${OPTARG};; + n) max_iperf_count=${OPTARG};; + P) iperf_p=${OPTARG};; + c) dest_ip=${OPTARG};; + p) iperf_base_port=${OPTARG};; + r) iperf_repeats=${OPTARG};; + esac +done +shift $((OPTIND -1)) + +netns_cmd= + +if [ ! -z "$netdev" ]; then + if [ ! -z "$ifaddr" ]; then + if [ -z "$netns" ]; then + netns=$netdev + fi + fi +else + echo "Interface name required" >&2 + exit -1 +fi + +if [ ! -z "$netns" ]; then + netns_cmd="ip netns exec $netns" + if [ -f "/var/run/netns/$netns" ]; then + echo "Network namespace '$netns' already exists" + else + echo "Creating network namespace '$netns'" + ip netns add $netns + echo "Adding interface '$netdev' to network namespace '$netns'" + ip link set dev $netdev netns $netns + $netns_cmd ip link set dev $netdev up + fi +fi + +if [ ! -z "$ifaddr" ]; then + echo "Adding address '$ifaddr' to '$netdev'" + $netns_cmd ip addr add $ifaddr dev $netdev +fi + +function cleanup() +{ + echo "Cleaning up..." + + # kill all subprocesses + trap '' TERM + pkill -P $$ + + # clean up netns + if [ ! -z "$netns" ]; then + if [ -f "/var/run/netns/$netns" -a -z "$(ip netns pids $netns)" ]; then + echo "Deleting network namespace '$netns'" + ip netns del $netns + fi + fi +} + +trap "exit" INT TERM +trap cleanup EXIT + +# run tests + +nic_info=$($netns_cmd $mqnic_dump -d $netdev) + +if [ $? -ne "0" ]; then + echo "Failed to communicate with device" >2 + exit -1 +fi + +fpga_id=$(echo "$nic_info" | grep "FPGA ID" | sed -n 's/^.*: //p') +fpga_part=$(echo "$nic_info" | grep "FPGA part" | sed -n 's/^.*: //p') +fw_id=$(echo "$nic_info" | grep "FW ID" | sed -n 's/^.*: //p') +fw_ver=$(echo "$nic_info" | grep "FW version" | sed -n 's/^.*: //p') +board_id=$(echo "$nic_info" | grep "Board ID" | sed -n 's/^.*: //p') +board_ver=$(echo "$nic_info" | grep "Board version" | sed -n 's/^.*: //p') +build_date=$(echo "$nic_info" | grep "Build date" | sed -n 's/^.*: //p') +git_hash=$(echo "$nic_info" | grep "Git hash" | sed -n 's/^.*: //p') +release_info=$(echo "$nic_info" | grep "Release info" | sed -n 's/^.*: //p') + +logname="$(date +%Y%m%dT%H%M%S)_${board_id}_fw${fw_ver}_${git_hash}" + +logdir=$base_logdir/$logname + +mkdir -p $logdir + +echo "$nic_info" > $logdir/mqnic_dump.log + +{ + +tests=() +declare -A test_desc +declare -A test_status +declare -A test_status_desc + +link_ok=0 + +echo "Firmware information" +echo "FPGA ID: $fpga_id" +echo "FPGA part: $fpga_part" +echo "FW ID: $fw_id" +echo "FW version: $fw_ver" +echo "Board ID: $board_id" +echo "Board version: $board_ver" +echo "Build date: $build_date" +echo "Git hash: $git_hash" +echo "Release info: $release_info" + +echo "CPU information from lscpu" +lscpu | tee $logdir/lscpu.log 2>&1 + +echo "Link information from iproute2" +$netns_cmd ip link show dev $netdev | tee $logdir/iproute2_link.log 2>&1 + +echo "Network device information from ethtool" +$netns_cmd ethtool $netdev | tee $logdir/ethtool.log 2>&1 + +echo "Network device features from ethtool" +$netns_cmd ethtool -k $netdev | tee $logdir/ethtool_k.log 2>&1 + + +tests+=("macaddr") +test_desc[macaddr]="Default MAC address" +test_status[macaddr]="PASS" +test_status_desc[macaddr]="OK" +echo "Current test: " ${test_desc[macaddr]} + +mac=$($netns_cmd ip link show dev $netdev | grep link/ether | tr -s ' ' | cut -d' ' -f 3) + +echo "Assigned MAC: $mac" + +b=$(echo $mac | cut -d : -f 1) + +if [ $((16#$b & 1)) -ne 0 ]; then + echo "Invalid assigned MAC (broadcast bit set)" + test_status[macaddr]="FAIL" + test_status_desc[macaddr]="Invalid MAC" +fi + +if [ $((16#$b & 2)) -ne 0 ]; then + echo "Locally assigned MAC; persistent MAC addressing not working or EEPROM is blank" + test_status[macaddr]="FAIL" + test_status_desc[macaddr]="Local MAC" +fi + +if [ "${test_status[macaddr]}" = "PASS" ]; then + echo "MAC address OK" +fi + + +tests+=("mod_i2c") +test_desc[mod_i2c]="Module communication" +test_status[mod_i2c]="PASS" +test_status_desc[mod_i2c]="OK" +echo "Current test: ${test_desc[mod_i2c]}" + +$netns_cmd ethtool -m $netdev | tee $logdir/ethtool_m.log 2>&1 + +if [ ${PIPESTATUS[0]} -ne 0 ]; then + echo "Failed to read module EEPROM" + test_status[mod_i2c]="FAIL" + test_status_desc[mod_i2c]="Failed to read EEPROM" +else + echo "Successfully read module EEPROM" +fi + + +tests+=("ping") +test_desc[ping]="Ping" +test_status[ping]="PASS" +test_status_desc[ping]="OK" +echo "Current test: ${test_desc[ping]}" + +$netns_cmd ping -c 4 -w 10 $dest_ip | tee $logdir/ping.log 2>&1 + +if [ ${PIPESTATUS[0]} -ne 0 ]; then + echo "Cannot communicate with test host" + test_status[ping]="FAIL" + test_status_desc[ping]="No response" + link_ok=0 +else + echo "Communication with test host OK" + link_ok=1 +fi + + +if [ $link_ok -ne 0 ]; then + echo "Link partner NIC MAC address from iproute2" + $netns_cmd ip neigh show $dest_ip | tee $logdir/iproute2_neigh.log 2>&1 +fi + + +ptp4l_pid=0 + +if [ $link_ok -ne 0 ]; then + echo "Start ptp4l" + + $netns_cmd ptp4l -i $netdev --slaveOnly=1 --tx_timestamp_timeout=10 -m > $logdir/ptp4l.log 2>&1 & + ptp4l_pid=$! +fi + + +tests+=("ptp_sync1") +test_desc[ptp_sync1]="PTP sync at idle" +test_status[ptp_sync1]="PASS" +test_status_desc[ptp_sync1]="OK" +echo "Current test: ${test_desc[ptp_sync1]}" + + +if [ $link_ok -ne 0 ]; then + + echo "Wait for PTP to sync" + tail -f $logdir/ptp4l.log & + tail_pid=$! + + sleep 60 + + kill $tail_pid + + ptp_status=$(cat $logdir/ptp4l.log | grep freq | grep rms | tail -1) + + echo $ptp_status + + ptp_offset_rms=$(echo "$ptp_status" | tr -s ' ' | cut -d ' ' -f 3) + + if [ ! -z "$ptp_offset_rms" ]; then + echo "RMS offset: $ptp_offset_rms" + + if [ "$ptp_offset_rms" -lt 100 ]; then + echo "PTP sync OK" + else + echo "PTP sync not sufficiently accurate" + echo "Note: this may caused by other components in the network not properly supporting PTP" + test_status[ptp_sync1]="FAIL" + test_status_desc[ptp_sync1]="Inaccurate sync" + fi + else + echo "PTP failed to sync" + test_status[ptp_sync1]="FAIL" + test_status_desc[ptp_sync1]="Did not sync" + fi + +else + echo "Skipping test due to link issue" + test_status[ptp_sync1]="SKIP" + test_status_desc[ptp_sync1]="No link" +fi + + +tests+=("bandwidth") +test_desc[bandwidth]="Bandwidth" +test_status[bandwidth]="PASS" +test_status_desc[bandwidth]="OK" +echo "Current test: ${test_desc[bandwidth]}" + + +if [ $link_ok -ne 0 ]; then + + $netns_cmd ./iperf_benchmark.sh --logdir "$logdir/iperf" -c $dest_ip -P $iperf_p -n $max_iperf_count -r $iperf_repeats | tee $logdir/iperf_benchmark.log 2>&1 + +else + echo "Skipping test due to link issue" + test_status[bandwidth]="SKIP" + test_status_desc[bandwidth]="No link" +fi + + +if [ $ptp4l_pid -ne 0 ]; then + echo "Stop ptp4l" + kill $ptp4l_pid +fi + + +tests+=("ptp_sync2") +test_desc[ptp_sync2]="PTP sync under load" +test_status[ptp_sync2]="PASS" +test_status_desc[ptp_sync2]="OK" +echo "Current test: ${test_desc[ptp_sync2]}" + + +if [ $link_ok -ne 0 ]; then + + ptp_status=$(cat $logdir/ptp4l.log | grep freq | grep rms | tail -1) + + echo $ptp_status + + ptp_offset_rms=$(echo "$ptp_status" | tr -s ' ' | cut -d ' ' -f 3) + + if [ ! -z "$ptp_offset_rms" ]; then + echo "RMS offset: $ptp_offset_rms" + + if [ "$ptp_offset_rms" -lt 100 ]; then + echo "PTP sync OK" + else + echo "PTP sync not sufficiently accurate" + echo "Note: this may caused by other components in the network not properly supporting PTP" + test_status[ptp_sync2]="FAIL" + test_status_desc[ptp_sync2]="Inaccurate sync" + fi + else + echo "PTP failed to sync" + test_status[ptp_sync2]="FAIL" + test_status_desc[ptp_sync2]="Did not sync" + fi + +else + echo "Skipping test due to link issue" + test_status[ptp_sync2]="SKIP" + test_status_desc[ptp_sync2]="No link" +fi + +echo + +{ + + echo "Test summary" + echo + echo "Date: $(date)" + echo + + echo "Design information" + echo + + echo " FPGA ID: $fpga_id" + echo " FPGA part: $fpga_part" + echo " FW ID: $fw_id" + echo " FW version: $fw_ver" + echo " Board ID: $board_id" + echo " Board version: $board_ver" + echo " Build date: $build_date" + echo " Git hash: $git_hash" + echo " Release info: $release_info" + echo + + echo "Test results" + echo + + overall="PASS" + + for test in "${tests[@]}"; do + printf " %-30s: %-4s (%s)\n" "${test_desc[$test]}" "${test_status[$test]}" "${test_status_desc[$test]}" + + if [ "${test_status[$test]}" = "FAIL" ]; then + overall="FAIL" + fi + done + + echo + echo "Overall result: $overall" + echo + +} | tee $logdir/summary.log 2>&1 + +echo "Done" + +} | tee $logdir/test.log 2>&1