diff --git a/Makefile b/Makefile index f9ce302..cdd1fad 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ PREFIX ?= /usr/local BINDIR := $(PREFIX)/bin APPDIR := $(PREFIX)/share/applications SYSTEMDDIR := /etc/systemd/system +CONFDIR := /etc LOGDIR := /var/log/freeze-watcher LOG_GROUP ?= $(shell id -gn) @@ -12,6 +13,13 @@ install: install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop + @if [ ! -e "$(DESTDIR)$(CONFDIR)/freeze-watcher.conf" ]; then \ + install -Dm644 config/freeze-watcher.conf $(DESTDIR)$(CONFDIR)/freeze-watcher.conf; \ + echo "Installed default config at $(CONFDIR)/freeze-watcher.conf"; \ + else \ + install -Dm644 config/freeze-watcher.conf $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new; \ + echo "Existing config preserved; new template at $(CONFDIR)/freeze-watcher.conf.new"; \ + fi install -dm775 -g $(LOG_GROUP) $(DESTDIR)$(LOGDIR) @echo "Installed. Run 'make enable' to start the service." @@ -20,7 +28,8 @@ uninstall: rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop - @echo "Uninstalled. Logs at $(LOGDIR) preserved." + rm -f $(DESTDIR)$(CONFDIR)/freeze-watcher.conf.new + @echo "Uninstalled. Config at $(CONFDIR)/freeze-watcher.conf and logs at $(LOGDIR) preserved." enable: systemctl daemon-reload diff --git a/README.md b/README.md index bc5819e..da42764 100644 --- a/README.md +++ b/README.md @@ -64,20 +64,33 @@ sudo systemctl restart freeze-watcher.service If you're using the Arch package: `./packaging/build-package.sh && sudo pacman -U *.pkg.tar.zst` will rebuild and upgrade in place. -## Tuning thresholds +## Configuration -Open `src/freeze-watcher.sh` and edit the constants at the top: +Edit `/etc/freeze-watcher.conf`. After changes: ``` -PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger -PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger -DSTATE_THRESHOLD=5 # processes in D state -BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat -LOAD_THRESHOLD=20 # 1-min load average -COOLDOWN=30 # seconds between captures -SAMPLE_INTERVAL=1 # seconds between samples +sudo systemctl restart freeze-watcher.service ``` +Options: + +| Variable | Default | Purpose | +|---|---|---| +| `PSI_IO_THRESHOLD` | 40 | io pressure avg10 % to trigger | +| `PSI_CPU_THRESHOLD` | 50 | cpu pressure avg10 % to trigger | +| `DSTATE_THRESHOLD` | 5 | processes in uninterruptible sleep | +| `BLOCKED_THRESHOLD` | 5 | procs_blocked from /proc/stat | +| `LOAD_THRESHOLD` | 20 | 1-min load average | +| `SAMPLE_INTERVAL` | 1 | seconds between samples | +| `COOLDOWN` | 30 | min seconds between captures | +| `MAX_SNAPSHOTS` | 200 | auto-delete oldest beyond this count (0 disables) | +| `MAX_SNAPSHOT_AGE_DAYS` | 30 | auto-delete older than this (0 disables) | +| `NOTIFY_USERS` | "" | comma-separated users to notify-send on capture | +| `NOTIFY_URGENCY` | normal | low / normal / critical | +| `CAPTURE_KERNEL_STACKS` | 1 | dump /proc/[pid]/stack for D-state procs | +| `CAPTURE_BPFTRACE` | 0 | run bpftrace one-shot (requires bpftrace) | +| `CAPTURE_CGROUP_PSI` | 1 | walk cgroups for top per-cgroup PSI | + ## GUI Launch from app menu (search "Freeze Watcher") or `freeze-monitor-gui`. @@ -105,13 +118,9 @@ Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention ( ## Future ideas -- Per-cgroup PSI capture -- eBPF tracing for long-running syscalls - Web UI option (no Qt requirement) -- Auto-prune old snapshots -- Configurable `/etc/freeze-watcher.conf` instead of inline thresholds -- Integration with desktop notifications via `notify-send` - Optional perf-profile capture during event +- HTML/JSON snapshot output mode ## License diff --git a/config/freeze-watcher.conf b/config/freeze-watcher.conf new file mode 100644 index 0000000..20bd6d2 --- /dev/null +++ b/config/freeze-watcher.conf @@ -0,0 +1,31 @@ +# freeze-watcher configuration +# Sourced by /usr/local/bin/freeze-watcher.sh at startup. +# Lines are bash variable assignments. Restart the service after editing: +# sudo systemctl restart freeze-watcher.service + +# --- thresholds ---------------------------------------------------------- +# A snapshot is captured when ANY of these is exceeded. +PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger +PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger +DSTATE_THRESHOLD=5 # processes in uninterruptible sleep +BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat +LOAD_THRESHOLD=20 # 1-min load average + +# --- timing -------------------------------------------------------------- +SAMPLE_INTERVAL=1 # seconds between samples +COOLDOWN=30 # min seconds between captures + +# --- retention ----------------------------------------------------------- +MAX_SNAPSHOTS=200 # auto-delete oldest beyond this count (0 = no cap) +MAX_SNAPSHOT_AGE_DAYS=30 # auto-delete older than this (0 = no age cap) + +# --- notifications ------------------------------------------------------- +# Send a desktop notification when a snapshot is captured. +# Comma-separated list of users to notify (must be logged into a graphical session). +NOTIFY_USERS="" # e.g. "luna" or "luna,alice" +NOTIFY_URGENCY=normal # low, normal, critical + +# --- capture options ----------------------------------------------------- +CAPTURE_KERNEL_STACKS=1 # dump /proc/[pid]/stack for D-state procs (free, very useful) +CAPTURE_BPFTRACE=0 # run bpftrace one-shot during capture (requires bpftrace) +CAPTURE_CGROUP_PSI=1 # walk /sys/fs/cgroup for top PSI cgroups diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index ae05c3b..56efb1e 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -1,16 +1,22 @@ # Maintainer: luna pkgname=freeze-watcher -pkgver=0.1.0 +pkgver=0.2.0 pkgrel=1 pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected" arch=('any') url="https://github.com/luna/freeze-watcher" license=('MIT') -depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux') -optdepends=('xdg-utils: open log directory from GUI') +depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux' 'libnotify' 'sudo') +optdepends=( + 'xdg-utils: open log directory from GUI' + 'bpftrace: kernel stack sampling during capture (CAPTURE_BPFTRACE=1)' +) source=("$pkgname-$pkgver.tar.gz::file://$PWD/../$pkgname-$pkgver.tar.gz") sha256sums=('SKIP') -backup=('etc/systemd/system/freeze-watcher.service') +backup=( + 'etc/systemd/system/freeze-watcher.service' + 'etc/freeze-watcher.conf' +) package() { cd "$srcdir/$pkgname-$pkgver" diff --git a/packaging/build-package.sh b/packaging/build-package.sh index 45acb26..e11d15e 100755 --- a/packaging/build-package.sh +++ b/packaging/build-package.sh @@ -14,7 +14,7 @@ trap "rm -rf $tmp" EXIT # Stage source as a tarball (PKGBUILD expects it as $pkgname-$pkgver.tar.gz) mkdir "$tmp/$PKGNAME-$PKGVER" -cp -r src systemd desktop Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true +cp -r src systemd desktop config Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true tar czf "$tmp/$PKGNAME-$PKGVER.tar.gz" -C "$tmp" "$PKGNAME-$PKGVER" cp packaging/PKGBUILD "$tmp/PKGBUILD" diff --git a/src/freeze-monitor-gui b/src/freeze-monitor-gui index a0e5821..f78c7db 100755 --- a/src/freeze-monitor-gui +++ b/src/freeze-monitor-gui @@ -149,6 +149,7 @@ class FreezeMonitor(QMainWindow): # Tray (created before timer so tick() can use it) self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self) + self.tray.setToolTip("Freeze Watcher") menu = QMenu() show_act = QAction("Show window", self) show_act.triggered.connect(self.showNormal) @@ -198,13 +199,19 @@ class FreezeMonitor(QMainWindow): self.metrics["Blocked"].set_value(blocked, 3, 6, "") self.metrics["Load (1m)"].set_value(int(load), 8, 16, "") - if service_active(): + active = service_active() + if active: self.status_label.setText("● Service: ACTIVE") self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;") else: self.status_label.setText("● Service: STOPPED") self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;") + self.tray.setToolTip( + f"Freeze Watcher — {'active' if active else 'stopped'}\n" + f"PSI io {int(psi_io)}% / cpu {int(psi_cpu)}% — D:{dstate} blk:{blocked} load:{load:.2f}" + ) + # Auto-refresh events list when a new file appears current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set() if current != self._known_events: diff --git a/src/freeze-watcher.sh b/src/freeze-watcher.sh index 31d3d2a..59d31f4 100755 --- a/src/freeze-watcher.sh +++ b/src/freeze-watcher.sh @@ -2,23 +2,116 @@ # Continuously samples system state. Writes a detailed snapshot when # anomaly thresholds are crossed (D-states, PSI pressure, blocked procs). +set -u + LOG_DIR=/var/log/freeze-watcher +CONFIG=/etc/freeze-watcher.conf + +# --- defaults (overridden by /etc/freeze-watcher.conf) ------------------- +PSI_IO_THRESHOLD=40 +PSI_CPU_THRESHOLD=50 +DSTATE_THRESHOLD=5 +BLOCKED_THRESHOLD=5 +LOAD_THRESHOLD=20 +COOLDOWN=30 +SAMPLE_INTERVAL=1 +MAX_SNAPSHOTS=200 +MAX_SNAPSHOT_AGE_DAYS=30 +NOTIFY_USERS="" +NOTIFY_URGENCY=normal +CAPTURE_KERNEL_STACKS=1 +CAPTURE_BPFTRACE=0 +CAPTURE_CGROUP_PSI=1 + +[ -f "$CONFIG" ] && . "$CONFIG" + mkdir -p "$LOG_DIR" -# Thresholds -PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger -PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger -DSTATE_THRESHOLD=5 # processes in D state -BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat -LOAD_THRESHOLD=20 # 1-min load average -COOLDOWN=30 # seconds between captures -SAMPLE_INTERVAL=1 # seconds between samples - last_capture=0 +captures_since_prune=0 + +# Send a desktop notification to each configured user. +# Daemon runs as root; we sudo into the user's session for DBus access. +notify_users() { + [ -z "$NOTIFY_USERS" ] && return 0 + local body="$1" + local IFS=, + for user in $NOTIFY_USERS; do + user=$(echo "$user" | tr -d '[:space:]') + [ -z "$user" ] && continue + local uid + uid=$(id -u "$user" 2>/dev/null) || continue + sudo -u "$user" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$uid/bus" \ + notify-send -u "$NOTIFY_URGENCY" -a "freeze-watcher" \ + "Freeze captured" "$body" 2>/dev/null & + done +} + +# Delete old snapshots based on MAX_SNAPSHOTS and MAX_SNAPSHOT_AGE_DAYS. +prune_snapshots() { + if [ "${MAX_SNAPSHOT_AGE_DAYS:-0}" -gt 0 ]; then + find "$LOG_DIR" -maxdepth 1 -name 'snap-*.log' -type f \ + -mtime "+$MAX_SNAPSHOT_AGE_DAYS" -delete 2>/dev/null + fi + if [ "${MAX_SNAPSHOTS:-0}" -gt 0 ]; then + local files + files=$(ls -1t "$LOG_DIR"/snap-*.log 2>/dev/null | tail -n "+$((MAX_SNAPSHOTS+1))") + [ -n "$files" ] && echo "$files" | xargs -r rm -f + fi +} + +capture_kernel_stacks() { + echo "## Kernel stack traces of D-state procs" + ps -eo pid,state --no-headers | awk '$2=="D"{print $1}' | while read -r pid; do + local comm + comm=$(cat "/proc/$pid/comm" 2>/dev/null) || continue + echo "--- pid=$pid comm=$comm ---" + cat "/proc/$pid/stack" 2>/dev/null | head -20 + done +} + +capture_cgroup_psi() { + echo "## Top cgroups by PSI io.some avg10" + { + for f in /sys/fs/cgroup/io.pressure \ + /sys/fs/cgroup/*/io.pressure \ + /sys/fs/cgroup/*/*/io.pressure; do + [ -r "$f" ] || continue + local v + v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null) + [ -z "$v" ] && continue + [ "$v" = "0.00" ] && continue + printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')" + done + } 2>/dev/null | sort -rn | head -10 + echo + echo "## Top cgroups by PSI cpu.some avg10" + { + for f in /sys/fs/cgroup/cpu.pressure \ + /sys/fs/cgroup/*/cpu.pressure \ + /sys/fs/cgroup/*/*/cpu.pressure; do + [ -r "$f" ] || continue + local v + v=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print $i; exit}}' "$f" 2>/dev/null) + [ -z "$v" ] && continue + [ "$v" = "0.00" ] && continue + printf "%s\t%s\n" "$v" "$(dirname "$f" | sed 's|/sys/fs/cgroup||')" + done + } 2>/dev/null | sort -rn | head -10 +} + +capture_bpftrace() { + command -v bpftrace >/dev/null 2>&1 || { echo "bpftrace not installed"; return; } + echo "## bpftrace: top kernel stacks for io_schedule waits over 2s" + timeout 3 bpftrace -e ' + kprobe:io_schedule { @[kstack(5)] = count(); } + interval:s:2 { exit(); } + ' 2>/dev/null | tail -40 +} capture_snapshot() { local reason="$1" - local now=$(date +%s) local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log" { echo "=== FREEZE EVENT TRIGGERED: $reason ===" @@ -35,9 +128,17 @@ capture_snapshot() { echo "[$k]"; cat /proc/pressure/$k 2>/dev/null done echo - echo "## D-state processes (with kernel wchan = where they're stuck)" + if [ "${CAPTURE_CGROUP_PSI:-1}" = "1" ]; then + capture_cgroup_psi + echo + fi + echo "## D-state processes (with kernel wchan)" ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30 echo + if [ "${CAPTURE_KERNEL_STACKS:-1}" = "1" ]; then + capture_kernel_stacks + echo + fi echo "## Top 15 by CPU" ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15 echo @@ -56,6 +157,10 @@ capture_snapshot() { echo "## softirq" cat /proc/softirqs echo + if [ "${CAPTURE_BPFTRACE:-0}" = "1" ]; then + capture_bpftrace + echo + fi echo "## Recent dmesg (last 50 lines)" dmesg -T | tail -50 echo @@ -63,24 +168,19 @@ capture_snapshot() { journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40 } > "$file" 2>&1 echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log" + notify_users "$reason" } echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log" +prune_snapshots while true; do now=$(date +%s) - # PSI pressure - parse "some avg10=X.XX" psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null) psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null) - - # D-state count dstate=$(ps -eo state --no-headers | grep -c "^D") - - # /proc/stat blocked blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat) - - # Load average load1=$(awk '{print int($1+0.5)}' /proc/loadavg) triggered="" @@ -93,6 +193,11 @@ while true; do if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then capture_snapshot "$triggered" & last_capture=$now + captures_since_prune=$((captures_since_prune+1)) + if [ "$captures_since_prune" -ge 20 ]; then + prune_snapshots & + captures_since_prune=0 + fi fi sleep "$SAMPLE_INTERVAL"