Skip to content

[show][generate_dump] Add optional tcpdump packet capture to "show techsupport" #4639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vaishnav-nexthop wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[show][generate_dump] Add optional tcpdump packet capture to "show techsupport" #4639

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 159 additions & 2 deletions scripts/generate_dump
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,17 @@ IS_SUPERVISOR=false

ENABLE_FLOW_DUMP=false

# tcpdump capture options
WITH_TCPDUMP=false
TCPDUMP_DURATION=60
TCPDUMP_PACKET_LIMIT=10000
TCPDUMP_FILTER=""
TCPDUMP_PID=""
# Track if tcpdump options were explicitly set (for validation)
TCPDUMP_DURATION_SET=false
TCPDUMP_PACKET_LIMIT_SET=false
TCPDUMP_FILTER_SET=false

# lock dirs/files
LOCKDIR="/tmp/techsupport-lock"
PIDFILE="${LOCKDIR}/PID"
Expand All @@ -69,6 +80,13 @@ rm_lock_and_exit()
handle_exit()
{
ECODE=$?
# Kill tcpdump if still running to prevent orphan processes
if [ ! -z "$TCPDUMP_PID" ]; then
echo "Killing tcpdump (PID: $TCPDUMP_PID)..." >&2
kill $TCPDUMP_PID 2>/dev/null || true
wait $TCPDUMP_PID 2>/dev/null || true
TCPDUMP_PID=""
fi
echo "Cleaning up working directory $TARDIR"
$RM -rf $TARDIR
echo "Removing lock. Exit: $ECODE" >&2
Expand Down Expand Up @@ -102,6 +120,85 @@ escape_quotes() {
echo $1 | sed 's/\"/\\\"/g'
}

###############################################################################
# Start tcpdump capture in background
# Uses tcpdump's native -G (rotate) and -W (file count limit) options for
# time-limited capture instead of external timeout command.
#
# Includes robustness checks:
# - Verifies tcpdump is available
# - Adds packet count limit to manage pcap file size
#
# Globals:
# LOGDIR, TCPDUMP_PID
# Arguments:
# $1 - duration: capture duration in seconds
# $2 - packet_limit: maximum number of packets to capture
# $3 - filter: BPF filter expression (optional)
# Returns:
# 0 on success, 1 on failure (sets TCPDUMP_PID global variable on success)
###############################################################################
start_tcpdump_capture() {
# Set ERR trap to invoke handle_error on any command failure within this function.
# This ensures errors are logged with exit code and line number for debugging.
trap 'handle_error $? $LINENO' ERR
local duration=$1
local packet_limit=$2
local filter=$3
local timestamp=$(date +%Y%m%d_%H%M%S)
local pcap_file="${LOGDIR}/tcpdump_${duration}s_${timestamp}.pcap"

# Verify tcpdump is available
if ! command -v tcpdump &> /dev/null; then
echo "Error: tcpdump not found, skipping capture"
return 1
fi

# Ensure LOGDIR exists
if [ ! -d "$LOGDIR" ]; then
$MKDIR $V -p "$LOGDIR"
fi

# Build tcpdump command
# -i any: Capture on all interfaces
# -G: Rotate dump file every <duration> seconds
# -W 1: Exit after 1 file (i.e., after <duration> seconds)
# -s 0: Capture full packets (no truncation)
# -c: Packet count limit to manage file size
local tcpdump_cmd=""
if [ -n "$filter" ]; then
tcpdump_cmd="tcpdump -i any -w ${pcap_file} -G ${duration} -W 1 -s 0 -c ${packet_limit} ${filter}"
echo "tcpdump: BPF filter: $filter"
else
tcpdump_cmd="tcpdump -i any -w ${pcap_file} -G ${duration} -W 1 -s 0 -c ${packet_limit}"
fi

echo "tcpdump: Running command: $tcpdump_cmd"
$tcpdump_cmd &
TCPDUMP_PID=$!
echo "tcpdump: Started capture (PID: $TCPDUMP_PID) for $duration seconds (limit: $packet_limit packets)"
echo "tcpdump: Output file: $pcap_file"
return 0
}

###############################################################################
# Wait for tcpdump to complete
# Globals:
# TCPDUMP_PID
# Arguments:
# None
# Returns:
# None
###############################################################################
wait_tcpdump_capture() {
if [ ! -z "$TCPDUMP_PID" ]; then
echo "Waiting for tcpdump (PID: $TCPDUMP_PID) to complete..."
wait $TCPDUMP_PID 2>/dev/null || true
echo "tcpdump capture completed"
TCPDUMP_PID=""
fi
}

save_bcmcmd() {
trap 'handle_error $? $LINENO' ERR
local start_t=$(date +%s%3N)
Expand Down Expand Up @@ -2633,6 +2730,11 @@ main() {
echo $BASE > $TECHSUPPORT_TIME_INFO
start_t=$(date +%s%3N)

# Start tcpdump capture if requested (runs in background during data collection)
if $WITH_TCPDUMP; then
start_tcpdump_capture "$TCPDUMP_DURATION" "$TCPDUMP_PACKET_LIMIT" "$TCPDUMP_FILTER"
fi

# Trigger BMC debug log dump task - Must be the first task to run
bmc_debug_log_dump_task_id=$(trigger_bmc_debug_log_dump)
if [ "$bmc_debug_log_dump_task_id" == "-1" ]; then
Expand Down Expand Up @@ -2865,9 +2967,17 @@ main() {

wait

# Wait for the tcpdump capture to finish BEFORE archiving. save_to_tar
# appends the dump tree by file content, so a capture still running at
# this point would be archived truncated (and the completed pcap, which
# is only ever added via this tree append, would never make it in).
if $WITH_TCPDUMP; then
wait_tcpdump_capture
fi

save_to_tar

save_sai_failure_dump
save_sai_failure_dump

if [[ "$asic" = "mellanox" ]] || [[ "$asic" = "nvidia-bluefield" ]]; then
collect_nvidia_sdk_dumps
Expand Down Expand Up @@ -3044,10 +3154,18 @@ OPTIONS
Collect the output of debug dump cli
-f
On DPU platforms, also collect a DPU flow dump
-T
Enable tcpdump capture during techsupport collection
-P PERIOD
Duration in seconds for tcpdump capture (default: 60). Requires -T
-L LIMIT
Maximum number of packets to capture (default: 10000). Requires -T
-F FILTER
BPF filter expression for tcpdump (e.g., 'port 179', 'udp port 3784'). Requires -T
EOF
}

while getopts ":xnvhzas:t:r:df" opt; do
while getopts ":xnvhzas:t:r:dfTP:L:F:" opt; do
case $opt in
x)
# enable bash debugging
Expand Down Expand Up @@ -3101,13 +3219,52 @@ while getopts ":xnvhzas:t:r:df" opt; do
f)
ENABLE_FLOW_DUMP=true
;;
T)
WITH_TCPDUMP=true
;;
P)
if ! [[ ${OPTARG} =~ ^[0-9]+$ ]]; then
echo "Invalid tcpdump duration value: ${OPTARG}, Please enter a numeric value."
exit $EXT_GENERAL
fi
TCPDUMP_DURATION="${OPTARG}"
TCPDUMP_DURATION_SET=true
;;
L)
if ! [[ ${OPTARG} =~ ^[0-9]+$ ]]; then
echo "Invalid tcpdump packet limit value: ${OPTARG}, Please enter a numeric value."
exit $EXT_GENERAL
fi
TCPDUMP_PACKET_LIMIT="${OPTARG}"
TCPDUMP_PACKET_LIMIT_SET=true
;;
F)
TCPDUMP_FILTER="${OPTARG}"
TCPDUMP_FILTER_SET=true
;;
/?)
echo "Invalid option: -$OPTARG" >&2
exit $EXT_GENERAL
;;
esac
done

# Validate that tcpdump options (-P, -L, -F) require -T to be specified
if ! $WITH_TCPDUMP; then
if $TCPDUMP_DURATION_SET; then
echo "Error: -P (tcpdump duration) requires -T (enable tcpdump) to be specified" >&2
exit $EXT_INVALID_ARGUMENT
fi
if $TCPDUMP_PACKET_LIMIT_SET; then
echo "Error: -L (tcpdump packet limit) requires -T (enable tcpdump) to be specified" >&2
exit $EXT_INVALID_ARGUMENT
fi
if $TCPDUMP_FILTER_SET; then
echo "Error: -F (tcpdump filter) requires -T (enable tcpdump) to be specified" >&2
exit $EXT_INVALID_ARGUMENT
fi
fi

# Check permissions before proceeding further
if [ `whoami` != root ] && ! $NOOP;
then
Expand Down
36 changes: 35 additions & 1 deletion show/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1796,9 +1796,34 @@ def users(verbose):
@click.option('--debug-dump', is_flag=True, help="Collect Debug Dump Output")
@click.option('--redirect-stderr', '-r', is_flag=True, help="Redirect an intermediate errors to STDERR")
@click.option('--flow-dump', is_flag=True, help="Collect DPU flow dump (Only valid on DPU platforms)")
@click.option('--with-tcpdump', is_flag=True, help="Capture traffic with tcpdump during techsupport")
@click.option('--tcpdump-duration', 'tcpdump_duration', type=click.IntRange(1, 300), default=None,
help="Duration in seconds for tcpdump capture (default: 60, range: 1-300). Requires --with-tcpdump")
@click.option('--tcpdump-packet-limit', 'tcpdump_packet_limit', type=click.IntRange(1, 100000), default=None,
help="Maximum number of packets to capture (default: 10000, range: 1-100000). Requires --with-tcpdump")
@click.option('--tcpdump-filter', 'tcpdump_filter', type=str, default=None,
help="BPF filter expression for tcpdump (e.g., 'port 179', 'udp port 3784'). Requires --with-tcpdump")
def techsupport(since, global_timeout, cmd_timeout, verbose, allow_process_stop,
silent, debug_dump, redirect_stderr, flow_dump):
silent, debug_dump, redirect_stderr, flow_dump,
with_tcpdump, tcpdump_duration, tcpdump_packet_limit, tcpdump_filter):
"""Gather information for troubleshooting"""

# Validate: tcpdump options require --with-tcpdump
if not with_tcpdump:
if tcpdump_duration is not None:
raise click.UsageError("--tcpdump-duration requires --with-tcpdump to be specified")
if tcpdump_packet_limit is not None:
raise click.UsageError("--tcpdump-packet-limit requires --with-tcpdump to be specified")
if tcpdump_filter is not None:
raise click.UsageError("--tcpdump-filter requires --with-tcpdump to be specified")

# Apply defaults when --with-tcpdump is used
if with_tcpdump:
if tcpdump_duration is None:
tcpdump_duration = 60 # default 60 seconds
if tcpdump_packet_limit is None:
tcpdump_packet_limit = 10000 # default 10k packets

cmd = ["sudo"]

if global_timeout:
Expand All @@ -1825,6 +1850,15 @@ def techsupport(since, global_timeout, cmd_timeout, verbose, allow_process_stop,
cmd += ['-t', str(cmd_timeout)]
if redirect_stderr:
cmd += ["-r"]

# Add tcpdump options
if with_tcpdump:
cmd += ['-T']
cmd += ['-P', str(tcpdump_duration)]
cmd += ['-L', str(tcpdump_packet_limit)]
if tcpdump_filter:
cmd += ['-F', tcpdump_filter]

run_command(cmd, display_cmd=verbose)


Expand Down
57 changes: 57 additions & 0 deletions tests/techsupport_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import os
import pytest
import show.main
from unittest.mock import patch, Mock
Expand All @@ -17,10 +18,66 @@
(['--debug-dump', '--redirect-stderr'], ['generate_dump', '-v', '-d', '-t', '5', '-r']),
(['--flow-dump'], ['generate_dump', '-v', '-f', '-t', '5']),
(['--debug-dump', '--flow-dump'], ['generate_dump', '-v', '-d', '-f', '-t', '5']),
# tcpdump: flag alone applies defaults (60s, 10k packets) -> -T -P -L
(['--with-tcpdump'],
['generate_dump', '-v', '-t', '5', '-T', '-P', '60', '-L', '10000']),
# tcpdump: all options forwarded, filter as -F
(['--with-tcpdump', '--tcpdump-duration', '30',
'--tcpdump-packet-limit', '500', '--tcpdump-filter', 'udp port 3784'],
['generate_dump', '-v', '-t', '5', '-T', '-P', '30', '-L', '500', '-F', 'udp port 3784']),
]
)
def test_techsupport(run_command, cli_arguments, expected):
runner = CliRunner()
result = runner.invoke(show.main.cli.commands['techsupport'], cli_arguments)
run_command.assert_called_with(EXPECTED_BASE_COMMAND + expected, display_cmd=False)


@patch("show.main.run_command")
@pytest.mark.parametrize(
"cli_arguments,expected_msg",
[
(['--tcpdump-duration', '30'], "--tcpdump-duration requires --with-tcpdump"),
(['--tcpdump-packet-limit', '500'], "--tcpdump-packet-limit requires --with-tcpdump"),
(['--tcpdump-filter', 'port 179'], "--tcpdump-filter requires --with-tcpdump"),
]
)
def test_techsupport_tcpdump_requires_flag(run_command, cli_arguments, expected_msg):
runner = CliRunner()
result = runner.invoke(show.main.cli.commands['techsupport'], cli_arguments)
assert result.exit_code != 0
assert expected_msg in result.output
run_command.assert_not_called()


@patch("show.main.run_command")
@pytest.mark.parametrize(
"cli_arguments",
[
['--with-tcpdump', '--tcpdump-duration', '0'], # below range (1-300)
['--with-tcpdump', '--tcpdump-duration', '301'], # above range
['--with-tcpdump', '--tcpdump-packet-limit', '0'], # below range (1-100000)
['--with-tcpdump', '--tcpdump-packet-limit', '100001'], # above range
]
)
def test_techsupport_tcpdump_out_of_range(run_command, cli_arguments):
runner = CliRunner()
result = runner.invoke(show.main.cli.commands['techsupport'], cli_arguments)
assert result.exit_code != 0
run_command.assert_not_called()


def test_tcpdump_awaited_before_save_to_tar():
"""The tcpdump pcap is written into the dump tree and is only added to the
archive by save_to_tar (a `tar -rhf` that snapshots file content). So the
capture must be awaited before save_to_tar, or a still-running capture is
archived truncated and the completed pcap is lost.
"""
script = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'generate_dump')
with open(script) as f:
lines = [ln.strip() for ln in f]
wait_idx = next(i for i, ln in enumerate(lines) if ln == 'wait_tcpdump_capture')
save_idx = next(i for i, ln in enumerate(lines) if ln == 'save_to_tar')
assert wait_idx < save_idx, (
"wait_tcpdump_capture (line {}) must precede save_to_tar (line {})".format(
wait_idx + 1, save_idx + 1))
Loading