luna/freeze-watcher
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

0.4.1: smarter NIC classifier (RPS-aware), 3x rapid D-state sampling for transient blocked spikes

Browse source
This commit is contained in:
luna 2026-05-10 22:05:04 -07:00
parent ea6a0ea675
commit ee0f0da59f
2 changed files with 33 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/freeze-watcher.sh
View file

@ -166,6 +166,8 @@ classify_snapshot() {
# NIC IRQ saturation — use delta against the running baseline so old
# cumulative skew doesn't trigger forever.
# With RPS enabled, per-flow concentration on one CPU is normal up to ~85%.
# Only flag when no RPS spreading is happening at all, or it's truly extreme.
if [ -r "$LOG_DIR/.softirq-baseline" ] && [ -n "$(echo "$softirq" | grep '^ *NET_RX:')" ]; then
local cur prev
cur=$(echo "$softirq" | grep '^ *NET_RX:' | sed 's/^ *NET_RX://')
@ -185,8 +187,20 @@ classify_snapshot() {
done
if [ "$total" -gt 100000 ]; then
local pct=$((max * 100 / total))
if [ "$pct" -gt 70 ]; then
diagnoses+=("NIC softirq concentrated on one CPU (${pct}% of recent NET_RX) — enable RPS, install irqbalance")
# Check if RPS is configured (any non-zero, non-single-bit value)
local rps_active=0
for rps in /sys/class/net/*/queues/rx-0/rps_cpus; do
[ -r "$rps" ] || continue
local v
v=$(cat "$rps" 2>/dev/null | tr -d ',')
# Strip leading zeros, count hex bits set
[ "$v" != "0" ] && [ "$v" != "00" ] && [ "$v" != "000" ] \
&& [ "$v" != "0000" ] && rps_active=1
done
if [ "$pct" -gt 90 ] && [ "$rps_active" = "0" ]; then
diagnoses+=("NIC softirq saturating one CPU (${pct}% of recent NET_RX, RPS disabled) — enable RPS, install irqbalance")
elif [ "$pct" -gt 95 ]; then
diagnoses+=("NIC softirq extreme concentration (${pct}% of recent NET_RX) — even with RPS, single-flow saturation; consider enabling RFS or reducing peer connections")
fi
fi
fi
@ -211,6 +225,17 @@ capture_snapshot() {
local body="$LOG_DIR/.snap-body-$$.tmp"
local final="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
# CAPTURE D-STATE FIRST - blocked spikes are often <1s long, so we need
# to grab process state before reading slower /proc/* files
local dstate_immediate
dstate_immediate=$(ps -eo state,pid,user,wchan:25,comm,args --no-headers 2>/dev/null | awk '$1=="D"' | head -30)
local dstate_t1
sleep 0.05
dstate_t1=$(ps -eo state,pid,user,wchan:25,comm,args --no-headers 2>/dev/null | awk '$1=="D"' | head -30)
local dstate_t2
sleep 0.05
dstate_t2=$(ps -eo state,pid,user,wchan:25,comm,args --no-headers 2>/dev/null | awk '$1=="D"' | head -30)
{
echo "## /proc/loadavg"
cat /proc/loadavg
@ -228,7 +253,11 @@ capture_snapshot() {
echo
fi
echo "## D-state processes (with kernel wchan)"
ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
# Use the snapshot taken before /proc reads to catch transient blockers.
# Combine 3 rapid samples (~100ms apart) and dedupe by pid.
echo "$dstate_immediate"
echo "$dstate_t1" | grep -vF "$dstate_immediate" 2>/dev/null
echo "$dstate_t2" | grep -vF "$dstate_immediate" | grep -vF "$dstate_t1" 2>/dev/null
echo
if [ "${CAPTURE_KERNEL_STACKS:-1}" = "1" ]; then
capture_kernel_stacks