0.3.0: auto-classifier, manual trigger via .trigger file, GUI tabs (graphs/snapshots/summary), filter, cross-snapshot correlation, NIC delta baseline
This commit is contained in:
luna
parent
41dcb0f577
commit
4c485b2af9
4 changed files with 561 additions and 81 deletions
|
|
@ -1,11 +1,12 @@
|
|||
#!/bin/bash
|
||||
# Continuously samples system state. Writes a detailed snapshot when
|
||||
# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
|
||||
# anomaly thresholds are crossed, or when the GUI requests a manual capture.
|
||||
|
||||
set -u
|
||||
|
||||
LOG_DIR=/var/log/freeze-watcher
|
||||
CONFIG=/etc/freeze-watcher.conf
|
||||
TRIGGER_FILE="$LOG_DIR/.trigger"
|
||||
|
||||
# --- defaults (overridden by /etc/freeze-watcher.conf) -------------------
|
||||
PSI_IO_THRESHOLD=40
|
||||
|
|
@ -30,8 +31,6 @@ mkdir -p "$LOG_DIR"
|
|||
last_capture=0
|
||||
captures_since_prune=0
|
||||
|
||||
# Send a desktop notification to each configured user.
|
||||
# Daemon runs as root; we sudo into the user's session for DBus access.
|
||||
notify_users() {
|
||||
[ -z "$NOTIFY_USERS" ] && return 0
|
||||
local body="$1"
|
||||
|
|
@ -48,7 +47,6 @@ notify_users() {
|
|||
done
|
||||
}
|
||||
|
||||
# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS.
|
||||
prune_snapshots() {
|
||||
if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then
|
||||
find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \
|
||||
|
|
@ -110,13 +108,108 @@ capture_bpftrace() {
|
|||
' 2>/dev/null | tail -40
|
||||
}
|
||||
|
||||
# Reads a snapshot file and emits one or more diagnosis lines.
|
||||
# Pattern-matches D-state wchans and PSI metrics against known freeze types.
|
||||
classify_snapshot() {
|
||||
local file="$1"
|
||||
local diagnoses=()
|
||||
|
||||
local dstate_section
|
||||
dstate_section=$(awk '/^## D-state processes/{flag=1; next} /^## /{flag=0} flag' "$file")
|
||||
local softirq
|
||||
softirq=$(awk '/^## softirq/{flag=1; next} /^## /{flag=0} flag' "$file")
|
||||
|
||||
local psi_io_some psi_io_full psi_cpu_some psi_mem_some
|
||||
psi_io_some=$(awk '/^\[io\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
psi_io_full=$(awk '/^\[io\]/{f=1; next} f && /^full/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
psi_cpu_some=$(awk '/^\[cpu\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
psi_mem_some=$(awk '/^\[memory\]/{f=1; next} f && /^some/{match($0,/avg10=[0-9.]+/); v=substr($0,RSTART+6,RLENGTH-6); split(v,a,"."); print a[1]+0; exit}' "$file")
|
||||
|
||||
local dcount
|
||||
dcount=$(echo "$dstate_section" | grep -c "^D ")
|
||||
|
||||
# btrfs lock contention
|
||||
if echo "$dstate_section" | grep -qE "btrfs_lock_root_node|btrfs_sync_log"; then
|
||||
if [ "${psi_io_some:-0}" -lt 10 ]; then
|
||||
diagnoses+=("btrfs metadata lock contention — wchans show btrfs_lock_root_node/sync_log with low PSI io. Typical of qgroups + many snapshots, or heavy fsync churn.")
|
||||
else
|
||||
diagnoses+=("btrfs lock contention with concurrent disk I/O")
|
||||
fi
|
||||
fi
|
||||
|
||||
# btrfs metadata pressure (less severe)
|
||||
if echo "$dstate_section" | grep -q "btrfs_search_slot" && \
|
||||
! echo "$dstate_section" | grep -q "btrfs_lock_root_node"; then
|
||||
diagnoses+=("btrfs metadata pressure — cleaner walking trees, fsyncs queueing")
|
||||
fi
|
||||
|
||||
# USB stall
|
||||
if echo "$dstate_section" | grep -qE "usb_sg_wait|scsi_eh|sd_check_events|usb_hcd"; then
|
||||
diagnoses+=("USB storage stall — one external drive responding slowly")
|
||||
fi
|
||||
|
||||
# Memory swap thrash
|
||||
if echo "$dstate_section" | grep -q "folio_wait_bit_common"; then
|
||||
if [ "${psi_mem_some:-0}" -gt 5 ]; then
|
||||
diagnoses+=("memory pressure / swap thrash — folio_wait + PSI mem ${psi_mem_some}%")
|
||||
else
|
||||
diagnoses+=("page-fault stall on swapped-out pages — apps faulting back from zram/swap (consider lower swappiness)")
|
||||
fi
|
||||
fi
|
||||
|
||||
# CPU oversubscription
|
||||
if [ "${psi_cpu_some:-0}" -gt 30 ] && [ "$dcount" -lt 3 ]; then
|
||||
diagnoses+=("CPU oversubscription — PSI cpu ${psi_cpu_some}%, few D-state procs. Heavy compile, transcoding, or runaway process competing for cores.")
|
||||
fi
|
||||
|
||||
# NIC IRQ saturation — use delta against the running baseline so old
|
||||
# cumulative skew doesn't trigger forever.
|
||||
if [ -r "$LOG_DIR/.softirq-baseline" ] && [ -n "$(echo "$softirq" | grep '^ *NET_RX:')" ]; then
|
||||
local cur prev
|
||||
cur=$(echo "$softirq" | grep '^ *NET_RX:' | sed 's/^ *NET_RX://')
|
||||
prev=$(awk '/^ *NET_RX:/{sub(/^ *NET_RX:/, ""); print; exit}' "$LOG_DIR/.softirq-baseline")
|
||||
if [ -n "$prev" ]; then
|
||||
local max=0 total=0 i=1
|
||||
for v in $cur; do
|
||||
[[ "$v" =~ ^[0-9]+$ ]] || { i=$((i+1)); continue; }
|
||||
local pv
|
||||
pv=$(echo "$prev" | awk -v n=$i '{print $n}')
|
||||
[[ "$pv" =~ ^[0-9]+$ ]] || pv=0
|
||||
local d=$((v - pv))
|
||||
[ "$d" -lt 0 ] && d=0
|
||||
[ "$d" -gt "$max" ] && max=$d
|
||||
total=$((total + d))
|
||||
i=$((i+1))
|
||||
done
|
||||
if [ "$total" -gt 100000 ]; then
|
||||
local pct=$((max * 100 / total))
|
||||
if [ "$pct" -gt 70 ]; then
|
||||
diagnoses+=("NIC softirq concentrated on one CPU (${pct}% of recent NET_RX) — enable RPS, install irqbalance")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Genuine I/O bottleneck
|
||||
if [ "${psi_io_full:-0}" -gt 30 ]; then
|
||||
diagnoses+=("genuine I/O bottleneck — PSI io full ${psi_io_full}%, device truly can't keep up")
|
||||
fi
|
||||
|
||||
if [ ${#diagnoses[@]} -eq 0 ]; then
|
||||
echo " • uncertain — no known signature matched; review wchans and metrics manually"
|
||||
else
|
||||
for d in "${diagnoses[@]}"; do
|
||||
echo " • $d"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
capture_snapshot() {
|
||||
local reason="$1"
|
||||
local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
local body="$LOG_DIR/.snap-body-$$.tmp"
|
||||
local final="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
echo "Time: $(date -Iseconds)"
|
||||
echo
|
||||
echo "## /proc/loadavg"
|
||||
cat /proc/loadavg
|
||||
echo
|
||||
|
|
@ -166,17 +259,47 @@ capture_snapshot() {
|
|||
echo
|
||||
echo "## journal warnings/errors last 60s"
|
||||
journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
|
||||
} > "$file" 2>&1
|
||||
echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
} > "$body" 2>&1
|
||||
|
||||
# Build final file: header + diagnosis (computed from body) + body
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
echo "Time: $(date -Iseconds)"
|
||||
echo
|
||||
echo "## Diagnosis (auto-classified)"
|
||||
classify_snapshot "$body"
|
||||
echo
|
||||
cat "$body"
|
||||
} > "$final"
|
||||
|
||||
rm -f "$body"
|
||||
echo "$(date -Iseconds) Captured snapshot to $final (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
notify_users "$reason"
|
||||
}
|
||||
|
||||
echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
|
||||
prune_snapshots
|
||||
|
||||
# Initial softirq baseline so classifier can compute deltas
|
||||
cp /proc/softirqs "$LOG_DIR/.softirq-baseline" 2>/dev/null
|
||||
softirq_baseline_at=$(date +%s)
|
||||
|
||||
while true; do
|
||||
now=$(date +%s)
|
||||
|
||||
# Manual trigger from GUI (file-based, no auth required for users in log group)
|
||||
if [ -f "$TRIGGER_FILE" ]; then
|
||||
manual_reason="manual"
|
||||
if [ -s "$TRIGGER_FILE" ]; then
|
||||
manual_reason=$(head -c 100 "$TRIGGER_FILE" | tr -d '\n')
|
||||
fi
|
||||
rm -f "$TRIGGER_FILE"
|
||||
capture_snapshot "$manual_reason" &
|
||||
last_capture=$now
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
|
||||
psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
|
||||
dstate=$(ps -eo state --no-headers | grep -c "^D")
|
||||
|
|
@ -200,5 +323,11 @@ while true; do
|
|||
fi
|
||||
fi
|
||||
|
||||
# Refresh softirq baseline every 5 minutes so classifier deltas reflect recent activity
|
||||
if [ $((now - softirq_baseline_at)) -ge 300 ]; then
|
||||
cp /proc/softirqs "$LOG_DIR/.softirq-baseline" 2>/dev/null
|
||||
softirq_baseline_at=$now
|
||||
fi
|
||||
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
done
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue