0.2.0: config file, auto-prune, notify-send, cgroup PSI, kernel stacks, bpftrace, tray tooltip
This commit is contained in:
luna
parent
d41f861802
commit
41dcb0f577
7 changed files with 206 additions and 39 deletions
11
Makefile
11
Makefile
|
|
@ -2,6 +2,7 @@ PREFIX ?= /usr/local
|
|||
BINDIR := $(PREFIX)/bin
|
||||
APPDIR := $(PREFIX)/share/applications
|
||||
SYSTEMDDIR := /etc/systemd/system
|
||||
CONFDIR := /etc
|
||||
LOGDIR := /var/log/freeze-watcher
|
||||
LOG_GROUP ?= $(shell id -gn)
|
||||
|
||||
|
|
@ -12,6 +13,13 @@ install:
|
|||
install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui
|
||||
install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
|
||||
install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
|
||||
@if [ ! -e "$(DESTDIR)$(CONFDIR)/freeze-watcher.conf" ]; then \
|
||||
install -Dm644 config/freeze-watcher.conf $(DESTDIR)$(CONFDIR)/freeze-watcher.conf; \
|
||||
echo "Installed default config at $(CONFDIR)/freeze-watcher.conf"; \
|
||||
else \
|
||||
install -Dm644 config/freeze-watcher.conf $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new; \
|
||||
echo "Existing config preserved; new template at $(CONFDIR)/freeze-watcher.conf.new"; \
|
||||
fi
|
||||
install -dm775 -g $(LOG_GROUP) $(DESTDIR)$(LOGDIR)
|
||||
@echo "Installed. Run 'make enable' to start the service."
|
||||
|
||||
|
|
@ -20,7 +28,8 @@ uninstall:
|
|||
rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui
|
||||
rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
|
||||
rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
|
||||
@echo "Uninstalled. Logs at $(LOGDIR) preserved."
|
||||
rm -f $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new
|
||||
@echo "Uninstalled. Config at $(CONFDIR)/freeze-watcher.conf and logs at $(LOGDIR) preserved."
|
||||
|
||||
enable:
|
||||
systemctl daemon-reload
|
||||
|
|
|
|||
37
README.md
37
README.md
|
|
@ -64,20 +64,33 @@ sudo systemctl restart freeze-watcher.service
|
|||
|
||||
If you're using the Arch package: `./packaging/build-package.sh && sudo pacman -U *.pkg.tar.zst` will rebuild and upgrade in place.
|
||||
|
||||
## Tuning thresholds
|
||||
## Configuration
|
||||
|
||||
Open `src/freeze-watcher.sh` and edit the constants at the top:
|
||||
Edit `/etc/freeze-watcher.conf`. After changes:
|
||||
|
||||
```
|
||||
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||
DSTATE_THRESHOLD=5 # processes in D state
|
||||
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||
LOAD_THRESHOLD=20 # 1-min load average
|
||||
COOLDOWN=30 # seconds between captures
|
||||
SAMPLE_INTERVAL=1 # seconds between samples
|
||||
sudo systemctl restart freeze-watcher.service
|
||||
```
|
||||
|
||||
Options:
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
|---|---|---|
|
||||
| `PSI_IO_THRESHOLD` | 40 | io pressure avg10 % to trigger |
|
||||
| `PSI_CPU_THRESHOLD` | 50 | cpu pressure avg10 % to trigger |
|
||||
| `DSTATE_THRESHOLD` | 5 | processes in uninterruptible sleep |
|
||||
| `BLOCKED_THRESHOLD` | 5 | procs_blocked from /proc/stat |
|
||||
| `LOAD_THRESHOLD` | 20 | 1-min load average |
|
||||
| `SAMPLE_INTERVAL` | 1 | seconds between samples |
|
||||
| `COOLDOWN` | 30 | min seconds between captures |
|
||||
| `MAX_SNAPSHOTS` | 200 | auto-delete oldest beyond this count (0 disables) |
|
||||
| `MAX_SNAPSHOT_AGE_DAYS` | 30 | auto-delete older than this (0 disables) |
|
||||
| `NOTIFY_USERS` | "" | comma-separated users to notify-send on capture |
|
||||
| `NOTIFY_URGENCY` | normal | low / normal / critical |
|
||||
| `CAPTURE_KERNEL_STACKS` | 1 | dump /proc/[pid]/stack for D-state procs |
|
||||
| `CAPTURE_BPFTRACE` | 0 | run bpftrace one-shot (requires bpftrace) |
|
||||
| `CAPTURE_CGROUP_PSI` | 1 | walk cgroups for top per-cgroup PSI |
|
||||
|
||||
## GUI
|
||||
|
||||
Launch from app menu (search "Freeze Watcher") or `freeze-monitor-gui`.
|
||||
|
|
@ -105,13 +118,9 @@ Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (
|
|||
|
||||
## Future ideas
|
||||
|
||||
- Per-cgroup PSI capture
|
||||
- eBPF tracing for long-running syscalls
|
||||
- Web UI option (no Qt requirement)
|
||||
- Auto-prune old snapshots
|
||||
- Configurable `/etc/freeze-watcher.conf` instead of inline thresholds
|
||||
- Integration with desktop notifications via `notify-send`
|
||||
- Optional perf-profile capture during event
|
||||
- HTML/JSON snapshot output mode
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
31
config/freeze-watcher.conf
Normal file
31
config/freeze-watcher.conf
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
# freeze-watcher configuration
|
||||
# Sourced by /usr/local/bin/freeze-watcher.sh at startup.
|
||||
# Lines are bash variable assignments. Restart the service after editing:
|
||||
# sudo systemctl restart freeze-watcher.service
|
||||
|
||||
# --- thresholds ----------------------------------------------------------
|
||||
# A snapshot is captured when ANY of these is exceeded.
|
||||
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||
DSTATE_THRESHOLD=5 # processes in uninterruptible sleep
|
||||
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||
LOAD_THRESHOLD=20 # 1-min load average
|
||||
|
||||
# --- timing --------------------------------------------------------------
|
||||
SAMPLE_INTERVAL=1 # seconds between samples
|
||||
COOLDOWN=30 # min seconds between captures
|
||||
|
||||
# --- retention -----------------------------------------------------------
|
||||
MAX_SNAPSHOTS=200 # auto-delete oldest beyond this count (0 = no cap)
|
||||
MAX_SNAPSHOT_AGE_DAYS=30 # auto-delete older than this (0 = no age cap)
|
||||
|
||||
# --- notifications -------------------------------------------------------
|
||||
# Send a desktop notification when a snapshot is captured.
|
||||
# Comma-separated list of users to notify (must be logged into a graphical session).
|
||||
NOTIFY_USERS="" # e.g. "luna" or "luna,alice"
|
||||
NOTIFY_URGENCY=normal # low, normal, critical
|
||||
|
||||
# --- capture options -----------------------------------------------------
|
||||
CAPTURE_KERNEL_STACKS=1 # dump /proc/[pid]/stack for D-state procs (free, very useful)
|
||||
CAPTURE_BPFTRACE=0 # run bpftrace one-shot during capture (requires bpftrace)
|
||||
CAPTURE_CGROUP_PSI=1 # walk /sys/fs/cgroup for top PSI cgroups
|
||||
|
|
@ -1,16 +1,22 @@
|
|||
# Maintainer: luna <anassaeneroi@pm.me>
|
||||
pkgname=freeze-watcher
|
||||
pkgver=0.1.0
|
||||
pkgver=0.2.0
|
||||
pkgrel=1
|
||||
pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected"
|
||||
arch=('any')
|
||||
url="https://github.com/luna/freeze-watcher"
|
||||
license=('MIT')
|
||||
depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux')
|
||||
optdepends=('xdg-utils: open log directory from GUI')
|
||||
depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux' 'libnotify' 'sudo')
|
||||
optdepends=(
|
||||
'xdg-utils: open log directory from GUI'
|
||||
'bpftrace: kernel stack sampling during capture (CAPTURE_BPFTRACE=1)'
|
||||
)
|
||||
source=("$pkgname-$pkgver.tar.gz::file://$PWD/../$pkgname-$pkgver.tar.gz")
|
||||
sha256sums=('SKIP')
|
||||
backup=('etc/systemd/system/freeze-watcher.service')
|
||||
backup=(
|
||||
'etc/systemd/system/freeze-watcher.service'
|
||||
'etc/freeze-watcher.conf'
|
||||
)
|
||||
|
||||
package() {
|
||||
cd "$srcdir/$pkgname-$pkgver"
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ trap "rm -rf $tmp" EXIT
|
|||
|
||||
# Stage source as a tarball (PKGBUILD expects it as $pkgname-$pkgver.tar.gz)
|
||||
mkdir "$tmp/$PKGNAME-$PKGVER"
|
||||
cp -r src systemd desktop Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true
|
||||
cp -r src systemd desktop config Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true
|
||||
tar czf "$tmp/$PKGNAME-$PKGVER.tar.gz" -C "$tmp" "$PKGNAME-$PKGVER"
|
||||
|
||||
cp packaging/PKGBUILD "$tmp/PKGBUILD"
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ class FreezeMonitor(QMainWindow):
|
|||
|
||||
# Tray (created before timer so tick() can use it)
|
||||
self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self)
|
||||
self.tray.setToolTip("Freeze Watcher")
|
||||
menu = QMenu()
|
||||
show_act = QAction("Show window", self)
|
||||
show_act.triggered.connect(self.showNormal)
|
||||
|
|
@ -198,13 +199,19 @@ class FreezeMonitor(QMainWindow):
|
|||
self.metrics["Blocked"].set_value(blocked, 3, 6, "")
|
||||
self.metrics["Load (1m)"].set_value(int(load), 8, 16, "")
|
||||
|
||||
if service_active():
|
||||
active = service_active()
|
||||
if active:
|
||||
self.status_label.setText("● Service: ACTIVE")
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;")
|
||||
else:
|
||||
self.status_label.setText("● Service: STOPPED")
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;")
|
||||
|
||||
self.tray.setToolTip(
|
||||
f"Freeze Watcher — {'active' if active else 'stopped'}\n"
|
||||
f"PSI io {int(psi_io)}% / cpu {int(psi_cpu)}% — D:{dstate} blk:{blocked} load:{load:.2f}"
|
||||
)
|
||||
|
||||
# Auto-refresh events list when a new file appears
|
||||
current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set()
|
||||
if current != self._known_events:
|
||||
|
|
|
|||
|
|
@ -2,23 +2,116 @@
|
|||
# Continuously samples system state. Writes a detailed snapshot when
|
||||
# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
|
||||
|
||||
set -u
|
||||
|
||||
LOG_DIR=/var/log/freeze-watcher
|
||||
CONFIG=/etc/freeze-watcher.conf
|
||||
|
||||
# --- defaults (overridden by /etc/freeze-watcher.conf) -------------------
|
||||
PSI_IO_THRESHOLD=40
|
||||
PSI_CPU_THRESHOLD=50
|
||||
DSTATE_THRESHOLD=5
|
||||
BLOCKED_THRESHOLD=5
|
||||
LOAD_THRESHOLD=20
|
||||
COOLDOWN=30
|
||||
SAMPLE_INTERVAL=1
|
||||
MAX_SNAPSHOTS=200
|
||||
MAX_SNAPSHOT_AGE_DAYS=30
|
||||
NOTIFY_USERS=""
|
||||
NOTIFY_URGENCY=normal
|
||||
CAPTURE_KERNEL_STACKS=1
|
||||
CAPTURE_BPFTRACE=0
|
||||
CAPTURE_CGROUP_PSI=1
|
||||
|
||||
[ -f "$CONFIG" ] && . "$CONFIG"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Thresholds
|
||||
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||
DSTATE_THRESHOLD=5 # processes in D state
|
||||
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||
LOAD_THRESHOLD=20 # 1-min load average
|
||||
COOLDOWN=30 # seconds between captures
|
||||
SAMPLE_INTERVAL=1 # seconds between samples
|
||||
|
||||
last_capture=0
|
||||
captures_since_prune=0
|
||||
|
||||
# Send a desktop notification to each configured user.
|
||||
# Daemon runs as root; we sudo into the user's session for DBus access.
|
||||
notify_users() {
|
||||
[ -z "$NOTIFY_USERS" ] && return 0
|
||||
local body="$1"
|
||||
local IFS=,
|
||||
for user in $NOTIFY_USERS; do
|
||||
user=$(echo "$user" | tr -d '[:space:]')
|
||||
[ -z "$user" ] && continue
|
||||
local uid
|
||||
uid=$(id -u "$user" 2>/dev/null) || continue
|
||||
sudo -u "$user" \
|
||||
DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$uid/bus" \
|
||||
notify-send -u "$NOTIFY_URGENCY" -a "freeze-watcher" \
|
||||
"Freeze captured" "$body" 2>/dev/null &
|
||||
done
|
||||
}
|
||||
|
||||
# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS.
|
||||
prune_snapshots() {
|
||||
if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then
|
||||
find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \
|
||||
-mtime "+$MAX_SNAPSHOT_AGE_DAYS" -delete 2>/dev/null
|
||||
fi
|
||||
if [ "${MAX_SNAPSHOTS:-0}" -gt 0 ]; then
|
||||
local files
|
||||
files=$(ls -1t "$LOG_DIR"/snap-*.log 2>/dev/null | tail -n "+$((MAX_SNAPSHOTS+1))")
|
||||
[ -n "$files" ] && echo "$files" | xargs -r rm -f
|
||||
fi
|
||||
}
|
||||
|
||||
capture_kernel_stacks() {
|
||||
echo "## Kernel stack traces of D-state procs"
|
||||
ps -eo pid,state --no-headers | awk '$2=="D"{print $1}' | while read -r pid; do
|
||||
local comm
|
||||
comm=$(cat "/proc/$pid/comm" 2>/dev/null) || continue
|
||||
echo "--- pid=$pid comm=$comm ---"
|
||||
cat "/proc/$pid/stack" 2>/dev/null | head -20
|
||||
done
|
||||
}
|
||||
|
||||
capture_cgroup_psi() {
|
||||
echo "## Top cgroups by PSI io.some avg10"
|
||||
{
|
||||
for f in /sys/fs/cgroup/io.pressure \
|
||||
/sys/fs/cgroup/*/io.pressure \
|
||||
/sys/fs/cgroup/*/*/io.pressure; do
|
||||
[ -r "$f" ] || continue
|
||||
local v
|
||||
v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null)
|
||||
[ -z "$v" ] && continue
|
||||
[ "$v" = "0.00" ] && continue
|
||||
printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')"
|
||||
done
|
||||
} 2>/dev/null | sort -rn | head -10
|
||||
echo
|
||||
echo "## Top cgroups by PSI cpu.some avg10"
|
||||
{
|
||||
for f in /sys/fs/cgroup/cpu.pressure \
|
||||
/sys/fs/cgroup/*/cpu.pressure \
|
||||
/sys/fs/cgroup/*/*/cpu.pressure; do
|
||||
[ -r "$f" ] || continue
|
||||
local v
|
||||
v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null)
|
||||
[ -z "$v" ] && continue
|
||||
[ "$v" = "0.00" ] && continue
|
||||
printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')"
|
||||
done
|
||||
} 2>/dev/null | sort -rn | head -10
|
||||
}
|
||||
|
||||
capture_bpftrace() {
|
||||
command -v bpftrace >/dev/null 2>&1 || { echo "bpftrace not installed"; return; }
|
||||
echo "## bpftrace: top kernel stacks for io_schedule waits over 2s"
|
||||
timeout 3 bpftrace -e '
|
||||
kprobe:io_schedule { @[kstack(5)] = count(); }
|
||||
interval:s:2 { exit(); }
|
||||
' 2>/dev/null | tail -40
|
||||
}
|
||||
|
||||
capture_snapshot() {
|
||||
local reason="$1"
|
||||
local now=$(date +%s)
|
||||
local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
|
|
@ -35,9 +128,17 @@ capture_snapshot() {
|
|||
echo "[$k]"; cat /proc/pressure/$k 2>/dev/null
|
||||
done
|
||||
echo
|
||||
echo "## D-state processes (with kernel wchan = where they're stuck)"
|
||||
if [ "${CAPTURE_CGROUP_PSI:-1}" = "1" ]; then
|
||||
capture_cgroup_psi
|
||||
echo
|
||||
fi
|
||||
echo "## D-state processes (with kernel wchan)"
|
||||
ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
|
||||
echo
|
||||
if [ "${CAPTURE_KERNEL_STACKS:-1}" = "1" ]; then
|
||||
capture_kernel_stacks
|
||||
echo
|
||||
fi
|
||||
echo "## Top 15 by CPU"
|
||||
ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15
|
||||
echo
|
||||
|
|
@ -56,6 +157,10 @@ capture_snapshot() {
|
|||
echo "## softirq"
|
||||
cat /proc/softirqs
|
||||
echo
|
||||
if [ "${CAPTURE_BPFTRACE:-0}" = "1" ]; then
|
||||
capture_bpftrace
|
||||
echo
|
||||
fi
|
||||
echo "## Recent dmesg (last 50 lines)"
|
||||
dmesg -T | tail -50
|
||||
echo
|
||||
|
|
@ -63,24 +168,19 @@ capture_snapshot() {
|
|||
journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
|
||||
} > "$file" 2>&1
|
||||
echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
notify_users "$reason"
|
||||
}
|
||||
|
||||
echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
|
||||
prune_snapshots
|
||||
|
||||
while true; do
|
||||
now=$(date +%s)
|
||||
|
||||
# PSI pressure - parse "some avg10=X.XX"
|
||||
psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
|
||||
psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
|
||||
|
||||
# D-state count
|
||||
dstate=$(ps -eo state --no-headers | grep -c "^D")
|
||||
|
||||
# /proc/stat blocked
|
||||
blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat)
|
||||
|
||||
# Load average
|
||||
load1=$(awk '{print int($1+0.5)}' /proc/loadavg)
|
||||
|
||||
triggered=""
|
||||
|
|
@ -93,6 +193,11 @@ while true; do
|
|||
if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then
|
||||
capture_snapshot "$triggered" &
|
||||
last_capture=$now
|
||||
captures_since_prune=$((captures_since_prune+1))
|
||||
if [ "$captures_since_prune" -ge 20 ]; then
|
||||
prune_snapshots &
|
||||
captures_since_prune=0
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue