From d41f8618020b3e89706fdab69ea26be7526846a1 Mon Sep 17 00:00:00 2001 From: luna Date: Sun, 10 May 2026 20:56:19 -0700 Subject: [PATCH] Initial commit: freeze-watcher daemon, GUI, systemd unit, PKGBUILD --- .gitignore | 14 ++ LICENSE | 21 +++ Makefile | 39 +++++ README.md | 118 ++++++++++++++ desktop/freeze-monitor.desktop | 9 ++ packaging/PKGBUILD | 39 +++++ packaging/build-package.sh | 26 ++++ src/freeze-monitor-gui | 275 +++++++++++++++++++++++++++++++++ src/freeze-watcher.sh | 99 ++++++++++++ systemd/freeze-watcher.service | 15 ++ 10 files changed, 655 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 desktop/freeze-monitor.desktop create mode 100644 packaging/PKGBUILD create mode 100755 packaging/build-package.sh create mode 100755 src/freeze-monitor-gui create mode 100755 src/freeze-watcher.sh create mode 100644 systemd/freeze-watcher.service diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..145d82e --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Build artifacts +*.pkg.tar.* +*.tar.gz +src/pkg/ +src/build/ + +# Editor +*.swp +*.swo +.DS_Store + +# Python +__pycache__/ +*.pyc diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..133fa37 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 luna + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f9ce302 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +PREFIX ?= /usr/local +BINDIR := $(PREFIX)/bin +APPDIR := $(PREFIX)/share/applications +SYSTEMDDIR := /etc/systemd/system +LOGDIR := /var/log/freeze-watcher +LOG_GROUP ?= $(shell id -gn) + +.PHONY: install uninstall enable disable status logs clean + +install: + install -Dm755 src/freeze-watcher.sh $(DESTDIR)$(BINDIR)/freeze-watcher.sh + install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui + install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service + install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop + install -dm775 -g $(LOG_GROUP) $(DESTDIR)$(LOGDIR) + @echo "Installed. Run 'make enable' to start the service." + +uninstall: + rm -f $(DESTDIR)$(BINDIR)/freeze-watcher.sh + rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui + rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service + rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop + @echo "Uninstalled. Logs at $(LOGDIR) preserved." + +enable: + systemctl daemon-reload + systemctl enable --now freeze-watcher.service + +disable: + systemctl disable --now freeze-watcher.service + +status: + systemctl status freeze-watcher.service --no-pager + +logs: + journalctl -u freeze-watcher.service -f + +clean: + @echo "Nothing to clean (no build artifacts)." diff --git a/README.md b/README.md new file mode 100644 index 0000000..bc5819e --- /dev/null +++ b/README.md @@ -0,0 +1,118 @@ +# freeze-watcher + +A systemd daemon that continuously samples Linux system state and dumps a detailed forensic snapshot whenever freeze-like conditions are detected (D-state spikes, PSI pressure, blocked processes, load avg). Includes a Qt GUI for live metrics and snapshot browsing. + +Built to diagnose intermittent system freezes that don't show up in normal resource monitors — specifically the kind where some apps stall but the mouse keeps moving. + +## What it captures + +Each snapshot is a single text file in `/var/log/freeze-watcher/snap-YYYYMMDD-HHMMSS.log` containing: + +- `loadavg`, `procs_running`, `procs_blocked`, `intr`, `ctxt` +- PSI cpu / io / memory pressure (`some` and `full` averages) +- D-state processes with kernel `wchan` (where exactly they're stuck) +- Top 15 by CPU and RSS +- `meminfo` summary +- `iostat -xt` snapshot +- Top interrupt sources by total + per-CPU breakdown +- Full `softirq` table +- Last 50 dmesg lines +- Last 60s of journal warnings/errors + +The wchan column is the key one — that's how you tell `btrfs_lock_root_node` lock contention from `usb_sg_wait` USB stalls from `folio_wait_bit_common` page-fault stalls. + +## Components + +| Path | What | +|---|---| +| `src/freeze-watcher.sh` | The daemon — samples every 1s, captures on threshold | +| `src/freeze-monitor-gui` | PySide6 GUI: live metrics + snapshot browser + tray icon | +| `systemd/freeze-watcher.service` | systemd unit | +| `desktop/freeze-monitor.desktop` | App launcher entry | + +## Install + +### Via Makefile (system-wide install to `/usr/local`) + +``` +sudo make install +sudo make enable +``` + +### As an Arch package + +``` +./packaging/build-package.sh +sudo pacman -U freeze-watcher-*.pkg.tar.zst +sudo systemctl enable --now freeze-watcher.service +``` + +## Update workflow + +When you change something: + +``` +# 1. Edit files in src/ or systemd/ +# 2. Reinstall +sudo make install + +# 3. If the daemon script changed, restart it +sudo systemctl restart freeze-watcher.service + +# 4. If the GUI changed, just relaunch it (it's a regular Qt app) +``` + +If you're using the Arch package: `./packaging/build-package.sh && sudo pacman -U *.pkg.tar.zst` will rebuild and upgrade in place. + +## Tuning thresholds + +Open `src/freeze-watcher.sh` and edit the constants at the top: + +``` +PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger +PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger +DSTATE_THRESHOLD=5 # processes in D state +BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat +LOAD_THRESHOLD=20 # 1-min load average +COOLDOWN=30 # seconds between captures +SAMPLE_INTERVAL=1 # seconds between samples +``` + +## GUI + +Launch from app menu (search "Freeze Watcher") or `freeze-monitor-gui`. + +- Live PSI / D-state / load metrics with green/orange/red color coding (1s refresh) +- Service status with start/stop buttons (uses `pkexec`) +- Captured event list — click any row for full snapshot view +- Closes to system tray; quit from the tray menu +- Notification when a new snapshot is captured + +## Reading a snapshot + +The most informative section is **D-state processes** — the `wchan` tells you what kernel function each process is blocked in. Common ones: + +| wchan | Meaning | +|---|---| +| `btrfs_lock_root_node` | btrfs metadata lock contention (often qgroups + snapshots) | +| `btrfs_search_slot` | btrfs tree walk waiting for lock | +| `btrfs_sync_log` | fsync waiting for transaction commit | +| `folio_wait_bit_common` | Waiting for memory page (often swap-in) | +| `usb_sg_wait` | USB device stall | +| `io_schedule` | Generic I/O wait | + +Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (low PSI) from genuine resource pressure (high PSI). + +## Future ideas + +- Per-cgroup PSI capture +- eBPF tracing for long-running syscalls +- Web UI option (no Qt requirement) +- Auto-prune old snapshots +- Configurable `/etc/freeze-watcher.conf` instead of inline thresholds +- Integration with desktop notifications via `notify-send` +- Optional perf-profile capture during event + +## License + +MIT diff --git a/desktop/freeze-monitor.desktop b/desktop/freeze-monitor.desktop new file mode 100644 index 0000000..de537e5 --- /dev/null +++ b/desktop/freeze-monitor.desktop @@ -0,0 +1,9 @@ +[Desktop Entry] +Type=Application +Name=Freeze Watcher +Comment=Monitor system freezes and view captured snapshots +Exec=/usr/local/bin/freeze-monitor-gui +Icon=utilities-system-monitor +Terminal=false +Categories=System;Monitor; +StartupNotify=true diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD new file mode 100644 index 0000000..ae05c3b --- /dev/null +++ b/packaging/PKGBUILD @@ -0,0 +1,39 @@ +# Maintainer: luna +pkgname=freeze-watcher +pkgver=0.1.0 +pkgrel=1 +pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected" +arch=('any') +url="https://github.com/luna/freeze-watcher" +license=('MIT') +depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux') +optdepends=('xdg-utils: open log directory from GUI') +source=("$pkgname-$pkgver.tar.gz::file://$PWD/../$pkgname-$pkgver.tar.gz") +sha256sums=('SKIP') +backup=('etc/systemd/system/freeze-watcher.service') + +package() { + cd "$srcdir/$pkgname-$pkgver" + make DESTDIR="$pkgdir" PREFIX=/usr install +} + +post_install() { + echo "==> Run 'sudo systemctl enable --now freeze-watcher.service' to start" + echo "==> Snapshots will be written to /var/log/freeze-watcher/" + echo "==> Launch the GUI from your app menu (Freeze Watcher) or 'freeze-monitor-gui'" +} + +post_upgrade() { + systemctl daemon-reload + if systemctl is-active --quiet freeze-watcher.service; then + echo "==> Restart the service to pick up changes:" + echo " sudo systemctl restart freeze-watcher.service" + fi +} + +post_remove() { + if systemctl is-enabled --quiet freeze-watcher.service 2>/dev/null; then + systemctl disable --now freeze-watcher.service + fi + echo "==> Logs at /var/log/freeze-watcher/ preserved. Remove manually if desired." +} diff --git a/packaging/build-package.sh b/packaging/build-package.sh new file mode 100755 index 0000000..45acb26 --- /dev/null +++ b/packaging/build-package.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Build a local Arch package from the current source tree. +# Usage: ./packaging/build-package.sh + +set -e + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PKGNAME=freeze-watcher +PKGVER=$(awk -F= '/^pkgver=/{print $2}' "$REPO_ROOT/packaging/PKGBUILD") + +cd "$REPO_ROOT" +tmp=$(mktemp -d) +trap "rm -rf $tmp" EXIT + +# Stage source as a tarball (PKGBUILD expects it as $pkgname-$pkgver.tar.gz) +mkdir "$tmp/$PKGNAME-$PKGVER" +cp -r src systemd desktop Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true +tar czf "$tmp/$PKGNAME-$PKGVER.tar.gz" -C "$tmp" "$PKGNAME-$PKGVER" + +cp packaging/PKGBUILD "$tmp/PKGBUILD" +cd "$tmp" +makepkg -f --noconfirm +mv "$PKGNAME-$PKGVER-"*.pkg.tar.* "$REPO_ROOT/" +echo +echo "Built: $REPO_ROOT/$(ls "$REPO_ROOT" | grep "^$PKGNAME-$PKGVER" | head -1)" +echo "Install with: sudo pacman -U " diff --git a/src/freeze-monitor-gui b/src/freeze-monitor-gui new file mode 100755 index 0000000..a0e5821 --- /dev/null +++ b/src/freeze-monitor-gui @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""GUI for freeze-watcher service. Shows live metrics and captured snapshots.""" + +import os +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path + +from PySide6.QtCore import Qt, QTimer +from PySide6.QtGui import QFont, QColor, QAction, QIcon +from PySide6.QtWidgets import ( + QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, + QLabel, QListWidget, QListWidgetItem, QTextEdit, QPushButton, + QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle +) + +LOG_DIR = Path("/var/log/freeze-watcher") +SERVICE = "freeze-watcher.service" + + +def read_psi(name): + try: + line = (Path("/proc/pressure") / name).read_text().splitlines()[0] + m = re.search(r"avg10=([\d.]+)", line) + return float(m.group(1)) if m else 0.0 + except Exception: + return 0.0 + + +def read_dstate_count(): + try: + out = subprocess.run( + ["ps", "-eo", "state", "--no-headers"], + capture_output=True, text=True, timeout=2, + ).stdout + return sum(1 for ln in out.splitlines() if ln.startswith("D")) + except Exception: + return 0 + + +def read_loadavg(): + try: + return float(Path("/proc/loadavg").read_text().split()[0]) + except Exception: + return 0.0 + + +def read_procs_blocked(): + try: + for ln in Path("/proc/stat").read_text().splitlines(): + if ln.startswith("procs_blocked"): + return int(ln.split()[1]) + except Exception: + pass + return 0 + + +def service_active(): + r = subprocess.run(["systemctl", "is-active", SERVICE], capture_output=True, text=True) + return r.stdout.strip() == "active" + + +class MetricLabel(QLabel): + def __init__(self, name): + super().__init__("--") + self.name = name + self.setAlignment(Qt.AlignCenter) + self.setMinimumHeight(48) + f = QFont() + f.setPointSize(18) + f.setBold(True) + self.setFont(f) + self._set_color("#888") + + def _set_color(self, hex_): + self.setStyleSheet( + f"background:#1e1e1e; color:{hex_}; border:1px solid #333; border-radius:6px; padding:6px;" + ) + + def set_value(self, value, warn, crit, suffix=""): + self.setText(f"{value}{suffix}") + if value >= crit: + self._set_color("#ff5555") + elif value >= warn: + self._set_color("#f5a623") + else: + self._set_color("#5cdb95") + + +class FreezeMonitor(QMainWindow): + def __init__(self): + super().__init__() + self.setWindowTitle("Freeze Watcher") + self.resize(1100, 700) + + central = QWidget() + self.setCentralWidget(central) + root = QVBoxLayout(central) + + # Status bar + self.status_label = QLabel() + self.status_label.setStyleSheet("padding:8px; font-weight:bold;") + root.addWidget(self.status_label) + + # Live metrics grid + metrics_box = QGroupBox("Live metrics (1s refresh)") + grid = QGridLayout(metrics_box) + labels = ["PSI IO %", "PSI CPU %", "PSI Mem %", "D-state", "Blocked", "Load (1m)"] + self.metrics = {} + for i, name in enumerate(labels): + grid.addWidget(QLabel(name, alignment=Qt.AlignCenter), 0, i) + m = MetricLabel(name) + self.metrics[name] = m + grid.addWidget(m, 1, i) + root.addWidget(metrics_box) + + # Buttons + btn_row = QHBoxLayout() + self.btn_start = QPushButton("Start service") + self.btn_stop = QPushButton("Stop service") + self.btn_refresh = QPushButton("Refresh events") + self.btn_open = QPushButton("Open log dir") + self.btn_clear = QPushButton("Delete all snapshots") + for b in (self.btn_start, self.btn_stop, self.btn_refresh, self.btn_open, self.btn_clear): + btn_row.addWidget(b) + btn_row.addStretch() + self.btn_start.clicked.connect(lambda: self._svc("start")) + self.btn_stop.clicked.connect(lambda: self._svc("stop")) + self.btn_refresh.clicked.connect(self.load_events) + self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)])) + self.btn_clear.clicked.connect(self.clear_snapshots) + root.addLayout(btn_row) + + # Splitter: list | detail + splitter = QSplitter(Qt.Horizontal) + self.event_list = QListWidget() + self.event_list.itemSelectionChanged.connect(self.show_selected) + splitter.addWidget(self.event_list) + self.detail = QTextEdit() + self.detail.setReadOnly(True) + self.detail.setFont(QFont("monospace")) + splitter.addWidget(self.detail) + splitter.setSizes([350, 750]) + root.addWidget(splitter, stretch=1) + + self._known_events = set() + + # Tray (created before timer so tick() can use it) + self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self) + menu = QMenu() + show_act = QAction("Show window", self) + show_act.triggered.connect(self.showNormal) + quit_act = QAction("Quit", self) + quit_act.triggered.connect(QApplication.quit) + menu.addAction(show_act) + menu.addAction(quit_act) + self.tray.setContextMenu(menu) + self.tray.activated.connect(self._tray_clicked) + self.tray.show() + + # Timer + self.timer = QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(1000) + self.load_events() + self.tick() + + def _tray_clicked(self, reason): + if reason == QSystemTrayIcon.Trigger: + if self.isVisible(): + self.hide() + else: + self.showNormal() + self.raise_() + + def closeEvent(self, ev): + ev.ignore() + self.hide() + + def _svc(self, action): + subprocess.run(["pkexec", "systemctl", action, SERVICE]) + self.tick() + + def tick(self): + psi_io = read_psi("io") + psi_cpu = read_psi("cpu") + psi_mem = read_psi("memory") + dstate = read_dstate_count() + blocked = read_procs_blocked() + load = read_loadavg() + + self.metrics["PSI IO %"].set_value(int(psi_io), 30, 60, "") + self.metrics["PSI CPU %"].set_value(int(psi_cpu), 30, 60, "") + self.metrics["PSI Mem %"].set_value(int(psi_mem), 20, 50, "") + self.metrics["D-state"].set_value(dstate, 3, 6, "") + self.metrics["Blocked"].set_value(blocked, 3, 6, "") + self.metrics["Load (1m)"].set_value(int(load), 8, 16, "") + + if service_active(): + self.status_label.setText("● Service: ACTIVE") + self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;") + else: + self.status_label.setText("● Service: STOPPED") + self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;") + + # Auto-refresh events list when a new file appears + current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set() + if current != self._known_events: + new = current - self._known_events + self._known_events = current + self.load_events() + for n in new: + self.tray.showMessage( + "Freeze Watcher", + f"Captured: {n}", + QSystemTrayIcon.Warning, + 5000, + ) + + def load_events(self): + self.event_list.clear() + if not LOG_DIR.exists(): + return + files = sorted(LOG_DIR.glob("snap-*.log"), key=lambda p: p.stat().st_mtime, reverse=True) + for p in files: + reason = "" + try: + first = p.read_text().splitlines()[0] + if "TRIGGERED:" in first: + reason = first.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip() + except Exception: + pass + ts = datetime.fromtimestamp(p.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S") + item = QListWidgetItem(f"{ts}\n {reason}") + item.setData(Qt.UserRole, str(p)) + self.event_list.addItem(item) + self._known_events = set(p.name for p in files) + + def show_selected(self): + items = self.event_list.selectedItems() + if not items: + return + path = items[0].data(Qt.UserRole) + try: + self.detail.setPlainText(Path(path).read_text()) + except Exception as e: + self.detail.setPlainText(f"Error: {e}") + + def clear_snapshots(self): + if not LOG_DIR.exists(): + return + from PySide6.QtWidgets import QMessageBox + if QMessageBox.question(self, "Delete", "Delete all captured snapshots?") != QMessageBox.Yes: + return + for p in LOG_DIR.glob("snap-*.log"): + try: + subprocess.run(["pkexec", "rm", str(p)]) + except Exception: + pass + self.load_events() + self.detail.clear() + + +def main(): + app = QApplication(sys.argv) + app.setQuitOnLastWindowClosed(False) + w = FreezeMonitor() + w.show() + sys.exit(app.exec()) + + +if __name__ == "__main__": + main() diff --git a/src/freeze-watcher.sh b/src/freeze-watcher.sh new file mode 100755 index 0000000..31d3d2a --- /dev/null +++ b/src/freeze-watcher.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Continuously samples system state. Writes a detailed snapshot when +# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs). + +LOG_DIR=/var/log/freeze-watcher +mkdir -p "$LOG_DIR" + +# Thresholds +PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger +PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger +DSTATE_THRESHOLD=5 # processes in D state +BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat +LOAD_THRESHOLD=20 # 1-min load average +COOLDOWN=30 # seconds between captures +SAMPLE_INTERVAL=1 # seconds between samples + +last_capture=0 + +capture_snapshot() { + local reason="$1" + local now=$(date +%s) + local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log" + { + echo "=== FREEZE EVENT TRIGGERED: $reason ===" + echo "Time: $(date -Iseconds)" + echo + echo "## /proc/loadavg" + cat /proc/loadavg + echo + echo "## /proc/stat (procs_running, procs_blocked)" + grep -E "procs_running|procs_blocked|intr|ctxt" /proc/stat | head -5 + echo + echo "## PSI cpu/io/memory" + for k in cpu io memory; do + echo "[$k]"; cat /proc/pressure/$k 2>/dev/null + done + echo + echo "## D-state processes (with kernel wchan = where they're stuck)" + ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30 + echo + echo "## Top 15 by CPU" + ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15 + echo + echo "## Top 15 by RSS memory" + ps -eo pid,user,pcpu,pmem,rss,comm --no-headers --sort=-rss | head -15 + echo + echo "## /proc/meminfo summary" + grep -E "MemTotal|MemFree|MemAvailable|Dirty|Writeback|Slab|Cached|Buffers" /proc/meminfo + echo + echo "## iostat (1s snapshot)" + iostat -xt 1 2 2>&1 | tail -50 + echo + echo "## interrupts (top by total)" + awk 'NR==1{next} {sum=0; for(i=2;i<=NF-2;i++) sum+=$i; if(sum>1000) print sum, $0}' /proc/interrupts | sort -rn | head -15 + echo + echo "## softirq" + cat /proc/softirqs + echo + echo "## Recent dmesg (last 50 lines)" + dmesg -T | tail -50 + echo + echo "## journal warnings/errors last 60s" + journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40 + } > "$file" 2>&1 + echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log" +} + +echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log" + +while true; do + now=$(date +%s) + + # PSI pressure - parse "some avg10=X.XX" + psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null) + psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null) + + # D-state count + dstate=$(ps -eo state --no-headers | grep -c "^D") + + # /proc/stat blocked + blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat) + + # Load average + load1=$(awk '{print int($1+0.5)}' /proc/loadavg) + + triggered="" + [ "${psi_io:-0}" -ge "$PSI_IO_THRESHOLD" ] && triggered="psi_io=$psi_io%" + [ "${psi_cpu:-0}" -ge "$PSI_CPU_THRESHOLD" ] && triggered="${triggered:+$triggered, }psi_cpu=$psi_cpu%" + [ "${dstate:-0}" -ge "$DSTATE_THRESHOLD" ] && triggered="${triggered:+$triggered, }dstate=$dstate" + [ "${blocked:-0}" -ge "$BLOCKED_THRESHOLD" ] && triggered="${triggered:+$triggered, }blocked=$blocked" + [ "${load1:-0}" -ge "$LOAD_THRESHOLD" ] && triggered="${triggered:+$triggered, }load1=$load1" + + if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then + capture_snapshot "$triggered" & + last_capture=$now + fi + + sleep "$SAMPLE_INTERVAL" +done diff --git a/systemd/freeze-watcher.service b/systemd/freeze-watcher.service new file mode 100644 index 0000000..eca6049 --- /dev/null +++ b/systemd/freeze-watcher.service @@ -0,0 +1,15 @@ +[Unit] +Description=Freeze Watcher - captures system state during anomalous load +After=multi-user.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/freeze-watcher.sh +Restart=on-failure +RestartSec=5 +Nice=-5 +IOSchedulingClass=realtime +IOSchedulingPriority=0 + +[Install] +WantedBy=multi-user.target