Initial commit: freeze-watcher daemon, GUI, systemd unit, PKGBUILD
This commit is contained in:
luna
commit
d41f861802
10 changed files with 655 additions and 0 deletions
14
.gitignore
vendored
Normal file
14
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# Build artifacts
|
||||
*.pkg.tar.*
|
||||
*.tar.gz
|
||||
src/pkg/
|
||||
src/build/
|
||||
|
||||
# Editor
|
||||
*.swp
|
||||
*.swo
|
||||
.DS_Store
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
21
LICENSE
Normal file
21
LICENSE
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2026 luna
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
39
Makefile
Normal file
39
Makefile
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
PREFIX ?= /usr/local
|
||||
BINDIR := $(PREFIX)/bin
|
||||
APPDIR := $(PREFIX)/share/applications
|
||||
SYSTEMDDIR := /etc/systemd/system
|
||||
LOGDIR := /var/log/freeze-watcher
|
||||
LOG_GROUP ?= $(shell id -gn)
|
||||
|
||||
.PHONY: install uninstall enable disable status logs clean
|
||||
|
||||
install:
|
||||
install -Dm755 src/freeze-watcher.sh $(DESTDIR)$(BINDIR)/freeze-watcher.sh
|
||||
install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui
|
||||
install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
|
||||
install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
|
||||
install -dm775 -g $(LOG_GROUP) $(DESTDIR)$(LOGDIR)
|
||||
@echo "Installed. Run 'make enable' to start the service."
|
||||
|
||||
uninstall:
|
||||
rm -f $(DESTDIR)$(BINDIR)/freeze-watcher.sh
|
||||
rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui
|
||||
rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
|
||||
rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
|
||||
@echo "Uninstalled. Logs at $(LOGDIR) preserved."
|
||||
|
||||
enable:
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now freeze-watcher.service
|
||||
|
||||
disable:
|
||||
systemctl disable --now freeze-watcher.service
|
||||
|
||||
status:
|
||||
systemctl status freeze-watcher.service --no-pager
|
||||
|
||||
logs:
|
||||
journalctl -u freeze-watcher.service -f
|
||||
|
||||
clean:
|
||||
@echo "Nothing to clean (no build artifacts)."
|
||||
118
README.md
Normal file
118
README.md
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
# freeze-watcher
|
||||
|
||||
A systemd daemon that continuously samples Linux system state and dumps a detailed forensic snapshot whenever freeze-like conditions are detected (D-state spikes, PSI pressure, blocked processes, load avg). Includes a Qt GUI for live metrics and snapshot browsing.
|
||||
|
||||
Built to diagnose intermittent system freezes that don't show up in normal resource monitors — specifically the kind where some apps stall but the mouse keeps moving.
|
||||
|
||||
## What it captures
|
||||
|
||||
Each snapshot is a single text file in `/var/log/freeze-watcher/snap-YYYYMMDD-HHMMSS.log` containing:
|
||||
|
||||
- `loadavg`, `procs_running`, `procs_blocked`, `intr`, `ctxt`
|
||||
- PSI cpu / io / memory pressure (`some` and `full` averages)
|
||||
- D-state processes with kernel `wchan` (where exactly they're stuck)
|
||||
- Top 15 by CPU and RSS
|
||||
- `meminfo` summary
|
||||
- `iostat -xt` snapshot
|
||||
- Top interrupt sources by total + per-CPU breakdown
|
||||
- Full `softirq` table
|
||||
- Last 50 dmesg lines
|
||||
- Last 60s of journal warnings/errors
|
||||
|
||||
The wchan column is the key one — that's how you tell `btrfs_lock_root_node` lock contention from `usb_sg_wait` USB stalls from `folio_wait_bit_common` page-fault stalls.
|
||||
|
||||
## Components
|
||||
|
||||
| Path | What |
|
||||
|---|---|
|
||||
| `src/freeze-watcher.sh` | The daemon — samples every 1s, captures on threshold |
|
||||
| `src/freeze-monitor-gui` | PySide6 GUI: live metrics + snapshot browser + tray icon |
|
||||
| `systemd/freeze-watcher.service` | systemd unit |
|
||||
| `desktop/freeze-monitor.desktop` | App launcher entry |
|
||||
|
||||
## Install
|
||||
|
||||
### Via Makefile (system-wide install to `/usr/local`)
|
||||
|
||||
```
|
||||
sudo make install
|
||||
sudo make enable
|
||||
```
|
||||
|
||||
### As an Arch package
|
||||
|
||||
```
|
||||
./packaging/build-package.sh
|
||||
sudo pacman -U freeze-watcher-*.pkg.tar.zst
|
||||
sudo systemctl enable --now freeze-watcher.service
|
||||
```
|
||||
|
||||
## Update workflow
|
||||
|
||||
When you change something:
|
||||
|
||||
```
|
||||
# 1. Edit files in src/ or systemd/
|
||||
# 2. Reinstall
|
||||
sudo make install
|
||||
|
||||
# 3. If the daemon script changed, restart it
|
||||
sudo systemctl restart freeze-watcher.service
|
||||
|
||||
# 4. If the GUI changed, just relaunch it (it's a regular Qt app)
|
||||
```
|
||||
|
||||
If you're using the Arch package: `./packaging/build-package.sh && sudo pacman -U *.pkg.tar.zst` will rebuild and upgrade in place.
|
||||
|
||||
## Tuning thresholds
|
||||
|
||||
Open `src/freeze-watcher.sh` and edit the constants at the top:
|
||||
|
||||
```
|
||||
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||
DSTATE_THRESHOLD=5 # processes in D state
|
||||
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||
LOAD_THRESHOLD=20 # 1-min load average
|
||||
COOLDOWN=30 # seconds between captures
|
||||
SAMPLE_INTERVAL=1 # seconds between samples
|
||||
```
|
||||
|
||||
## GUI
|
||||
|
||||
Launch from app menu (search "Freeze Watcher") or `freeze-monitor-gui`.
|
||||
|
||||
- Live PSI / D-state / load metrics with green/orange/red color coding (1s refresh)
|
||||
- Service status with start/stop buttons (uses `pkexec`)
|
||||
- Captured event list — click any row for full snapshot view
|
||||
- Closes to system tray; quit from the tray menu
|
||||
- Notification when a new snapshot is captured
|
||||
|
||||
## Reading a snapshot
|
||||
|
||||
The most informative section is **D-state processes** — the `wchan` tells you what kernel function each process is blocked in. Common ones:
|
||||
|
||||
| wchan | Meaning |
|
||||
|---|---|
|
||||
| `btrfs_lock_root_node` | btrfs metadata lock contention (often qgroups + snapshots) |
|
||||
| `btrfs_search_slot` | btrfs tree walk waiting for lock |
|
||||
| `btrfs_sync_log` | fsync waiting for transaction commit |
|
||||
| `folio_wait_bit_common` | Waiting for memory page (often swap-in) |
|
||||
| `usb_sg_wait` | USB device stall |
|
||||
| `io_schedule` | Generic I/O wait |
|
||||
|
||||
Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (low PSI) from genuine resource pressure (high PSI).
|
||||
|
||||
## Future ideas
|
||||
|
||||
- Per-cgroup PSI capture
|
||||
- eBPF tracing for long-running syscalls
|
||||
- Web UI option (no Qt requirement)
|
||||
- Auto-prune old snapshots
|
||||
- Configurable `/etc/freeze-watcher.conf` instead of inline thresholds
|
||||
- Integration with desktop notifications via `notify-send`
|
||||
- Optional perf-profile capture during event
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
9
desktop/freeze-monitor.desktop
Normal file
9
desktop/freeze-monitor.desktop
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
[Desktop Entry]
|
||||
Type=Application
|
||||
Name=Freeze Watcher
|
||||
Comment=Monitor system freezes and view captured snapshots
|
||||
Exec=/usr/local/bin/freeze-monitor-gui
|
||||
Icon=utilities-system-monitor
|
||||
Terminal=false
|
||||
Categories=System;Monitor;
|
||||
StartupNotify=true
|
||||
39
packaging/PKGBUILD
Normal file
39
packaging/PKGBUILD
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
# Maintainer: luna <anassaeneroi@pm.me>
|
||||
pkgname=freeze-watcher
|
||||
pkgver=0.1.0
|
||||
pkgrel=1
|
||||
pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected"
|
||||
arch=('any')
|
||||
url="https://github.com/luna/freeze-watcher"
|
||||
license=('MIT')
|
||||
depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux')
|
||||
optdepends=('xdg-utils: open log directory from GUI')
|
||||
source=("$pkgname-$pkgver.tar.gz::file://$PWD/../$pkgname-$pkgver.tar.gz")
|
||||
sha256sums=('SKIP')
|
||||
backup=('etc/systemd/system/freeze-watcher.service')
|
||||
|
||||
package() {
|
||||
cd "$srcdir/$pkgname-$pkgver"
|
||||
make DESTDIR="$pkgdir" PREFIX=/usr install
|
||||
}
|
||||
|
||||
post_install() {
|
||||
echo "==> Run 'sudo systemctl enable --now freeze-watcher.service' to start"
|
||||
echo "==> Snapshots will be written to /var/log/freeze-watcher/"
|
||||
echo "==> Launch the GUI from your app menu (Freeze Watcher) or 'freeze-monitor-gui'"
|
||||
}
|
||||
|
||||
post_upgrade() {
|
||||
systemctl daemon-reload
|
||||
if systemctl is-active --quiet freeze-watcher.service; then
|
||||
echo "==> Restart the service to pick up changes:"
|
||||
echo " sudo systemctl restart freeze-watcher.service"
|
||||
fi
|
||||
}
|
||||
|
||||
post_remove() {
|
||||
if systemctl is-enabled --quiet freeze-watcher.service 2>/dev/null; then
|
||||
systemctl disable --now freeze-watcher.service
|
||||
fi
|
||||
echo "==> Logs at /var/log/freeze-watcher/ preserved. Remove manually if desired."
|
||||
}
|
||||
26
packaging/build-package.sh
Executable file
26
packaging/build-package.sh
Executable file
|
|
@ -0,0 +1,26 @@
|
|||
#!/bin/bash
|
||||
# Build a local Arch package from the current source tree.
|
||||
# Usage: ./packaging/build-package.sh
|
||||
|
||||
set -e
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
PKGNAME=freeze-watcher
|
||||
PKGVER=$(awk -F= '/^pkgver=/{print $2}' "$REPO_ROOT/packaging/PKGBUILD")
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
tmp=$(mktemp -d)
|
||||
trap "rm -rf $tmp" EXIT
|
||||
|
||||
# Stage source as a tarball (PKGBUILD expects it as $pkgname-$pkgver.tar.gz)
|
||||
mkdir "$tmp/$PKGNAME-$PKGVER"
|
||||
cp -r src systemd desktop Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true
|
||||
tar czf "$tmp/$PKGNAME-$PKGVER.tar.gz" -C "$tmp" "$PKGNAME-$PKGVER"
|
||||
|
||||
cp packaging/PKGBUILD "$tmp/PKGBUILD"
|
||||
cd "$tmp"
|
||||
makepkg -f --noconfirm
|
||||
mv "$PKGNAME-$PKGVER-"*.pkg.tar.* "$REPO_ROOT/"
|
||||
echo
|
||||
echo "Built: $REPO_ROOT/$(ls "$REPO_ROOT" | grep "^$PKGNAME-$PKGVER" | head -1)"
|
||||
echo "Install with: sudo pacman -U <path-to-pkg>"
|
||||
275
src/freeze-monitor-gui
Executable file
275
src/freeze-monitor-gui
Executable file
|
|
@ -0,0 +1,275 @@
|
|||
#!/usr/bin/env python3
|
||||
"""GUI for freeze-watcher service. Shows live metrics and captured snapshots."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from PySide6.QtCore import Qt, QTimer
|
||||
from PySide6.QtGui import QFont, QColor, QAction, QIcon
|
||||
from PySide6.QtWidgets import (
|
||||
QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
|
||||
QLabel, QListWidget, QListWidgetItem, QTextEdit, QPushButton,
|
||||
QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle
|
||||
)
|
||||
|
||||
LOG_DIR = Path("/var/log/freeze-watcher")
|
||||
SERVICE = "freeze-watcher.service"
|
||||
|
||||
|
||||
def read_psi(name):
|
||||
try:
|
||||
line = (Path("/proc/pressure") / name).read_text().splitlines()[0]
|
||||
m = re.search(r"avg10=([\d.]+)", line)
|
||||
return float(m.group(1)) if m else 0.0
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def read_dstate_count():
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["ps", "-eo", "state", "--no-headers"],
|
||||
capture_output=True, text=True, timeout=2,
|
||||
).stdout
|
||||
return sum(1 for ln in out.splitlines() if ln.startswith("D"))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def read_loadavg():
|
||||
try:
|
||||
return float(Path("/proc/loadavg").read_text().split()[0])
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def read_procs_blocked():
|
||||
try:
|
||||
for ln in Path("/proc/stat").read_text().splitlines():
|
||||
if ln.startswith("procs_blocked"):
|
||||
return int(ln.split()[1])
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def service_active():
|
||||
r = subprocess.run(["systemctl", "is-active", SERVICE], capture_output=True, text=True)
|
||||
return r.stdout.strip() == "active"
|
||||
|
||||
|
||||
class MetricLabel(QLabel):
|
||||
def __init__(self, name):
|
||||
super().__init__("--")
|
||||
self.name = name
|
||||
self.setAlignment(Qt.AlignCenter)
|
||||
self.setMinimumHeight(48)
|
||||
f = QFont()
|
||||
f.setPointSize(18)
|
||||
f.setBold(True)
|
||||
self.setFont(f)
|
||||
self._set_color("#888")
|
||||
|
||||
def _set_color(self, hex_):
|
||||
self.setStyleSheet(
|
||||
f"background:#1e1e1e; color:{hex_}; border:1px solid #333; border-radius:6px; padding:6px;"
|
||||
)
|
||||
|
||||
def set_value(self, value, warn, crit, suffix=""):
|
||||
self.setText(f"{value}{suffix}")
|
||||
if value >= crit:
|
||||
self._set_color("#ff5555")
|
||||
elif value >= warn:
|
||||
self._set_color("#f5a623")
|
||||
else:
|
||||
self._set_color("#5cdb95")
|
||||
|
||||
|
||||
class FreezeMonitor(QMainWindow):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.setWindowTitle("Freeze Watcher")
|
||||
self.resize(1100, 700)
|
||||
|
||||
central = QWidget()
|
||||
self.setCentralWidget(central)
|
||||
root = QVBoxLayout(central)
|
||||
|
||||
# Status bar
|
||||
self.status_label = QLabel()
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold;")
|
||||
root.addWidget(self.status_label)
|
||||
|
||||
# Live metrics grid
|
||||
metrics_box = QGroupBox("Live metrics (1s refresh)")
|
||||
grid = QGridLayout(metrics_box)
|
||||
labels = ["PSI IO %", "PSI CPU %", "PSI Mem %", "D-state", "Blocked", "Load (1m)"]
|
||||
self.metrics = {}
|
||||
for i, name in enumerate(labels):
|
||||
grid.addWidget(QLabel(name, alignment=Qt.AlignCenter), 0, i)
|
||||
m = MetricLabel(name)
|
||||
self.metrics[name] = m
|
||||
grid.addWidget(m, 1, i)
|
||||
root.addWidget(metrics_box)
|
||||
|
||||
# Buttons
|
||||
btn_row = QHBoxLayout()
|
||||
self.btn_start = QPushButton("Start service")
|
||||
self.btn_stop = QPushButton("Stop service")
|
||||
self.btn_refresh = QPushButton("Refresh events")
|
||||
self.btn_open = QPushButton("Open log dir")
|
||||
self.btn_clear = QPushButton("Delete all snapshots")
|
||||
for b in (self.btn_start, self.btn_stop, self.btn_refresh, self.btn_open, self.btn_clear):
|
||||
btn_row.addWidget(b)
|
||||
btn_row.addStretch()
|
||||
self.btn_start.clicked.connect(lambda: self._svc("start"))
|
||||
self.btn_stop.clicked.connect(lambda: self._svc("stop"))
|
||||
self.btn_refresh.clicked.connect(self.load_events)
|
||||
self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)]))
|
||||
self.btn_clear.clicked.connect(self.clear_snapshots)
|
||||
root.addLayout(btn_row)
|
||||
|
||||
# Splitter: list | detail
|
||||
splitter = QSplitter(Qt.Horizontal)
|
||||
self.event_list = QListWidget()
|
||||
self.event_list.itemSelectionChanged.connect(self.show_selected)
|
||||
splitter.addWidget(self.event_list)
|
||||
self.detail = QTextEdit()
|
||||
self.detail.setReadOnly(True)
|
||||
self.detail.setFont(QFont("monospace"))
|
||||
splitter.addWidget(self.detail)
|
||||
splitter.setSizes([350, 750])
|
||||
root.addWidget(splitter, stretch=1)
|
||||
|
||||
self._known_events = set()
|
||||
|
||||
# Tray (created before timer so tick() can use it)
|
||||
self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self)
|
||||
menu = QMenu()
|
||||
show_act = QAction("Show window", self)
|
||||
show_act.triggered.connect(self.showNormal)
|
||||
quit_act = QAction("Quit", self)
|
||||
quit_act.triggered.connect(QApplication.quit)
|
||||
menu.addAction(show_act)
|
||||
menu.addAction(quit_act)
|
||||
self.tray.setContextMenu(menu)
|
||||
self.tray.activated.connect(self._tray_clicked)
|
||||
self.tray.show()
|
||||
|
||||
# Timer
|
||||
self.timer = QTimer()
|
||||
self.timer.timeout.connect(self.tick)
|
||||
self.timer.start(1000)
|
||||
self.load_events()
|
||||
self.tick()
|
||||
|
||||
def _tray_clicked(self, reason):
|
||||
if reason == QSystemTrayIcon.Trigger:
|
||||
if self.isVisible():
|
||||
self.hide()
|
||||
else:
|
||||
self.showNormal()
|
||||
self.raise_()
|
||||
|
||||
def closeEvent(self, ev):
|
||||
ev.ignore()
|
||||
self.hide()
|
||||
|
||||
def _svc(self, action):
|
||||
subprocess.run(["pkexec", "systemctl", action, SERVICE])
|
||||
self.tick()
|
||||
|
||||
def tick(self):
|
||||
psi_io = read_psi("io")
|
||||
psi_cpu = read_psi("cpu")
|
||||
psi_mem = read_psi("memory")
|
||||
dstate = read_dstate_count()
|
||||
blocked = read_procs_blocked()
|
||||
load = read_loadavg()
|
||||
|
||||
self.metrics["PSI IO %"].set_value(int(psi_io), 30, 60, "")
|
||||
self.metrics["PSI CPU %"].set_value(int(psi_cpu), 30, 60, "")
|
||||
self.metrics["PSI Mem %"].set_value(int(psi_mem), 20, 50, "")
|
||||
self.metrics["D-state"].set_value(dstate, 3, 6, "")
|
||||
self.metrics["Blocked"].set_value(blocked, 3, 6, "")
|
||||
self.metrics["Load (1m)"].set_value(int(load), 8, 16, "")
|
||||
|
||||
if service_active():
|
||||
self.status_label.setText("● Service: ACTIVE")
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;")
|
||||
else:
|
||||
self.status_label.setText("● Service: STOPPED")
|
||||
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;")
|
||||
|
||||
# Auto-refresh events list when a new file appears
|
||||
current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set()
|
||||
if current != self._known_events:
|
||||
new = current - self._known_events
|
||||
self._known_events = current
|
||||
self.load_events()
|
||||
for n in new:
|
||||
self.tray.showMessage(
|
||||
"Freeze Watcher",
|
||||
f"Captured: {n}",
|
||||
QSystemTrayIcon.Warning,
|
||||
5000,
|
||||
)
|
||||
|
||||
def load_events(self):
|
||||
self.event_list.clear()
|
||||
if not LOG_DIR.exists():
|
||||
return
|
||||
files = sorted(LOG_DIR.glob("snap-*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
for p in files:
|
||||
reason = ""
|
||||
try:
|
||||
first = p.read_text().splitlines()[0]
|
||||
if "TRIGGERED:" in first:
|
||||
reason = first.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip()
|
||||
except Exception:
|
||||
pass
|
||||
ts = datetime.fromtimestamp(p.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S")
|
||||
item = QListWidgetItem(f"{ts}\n {reason}")
|
||||
item.setData(Qt.UserRole, str(p))
|
||||
self.event_list.addItem(item)
|
||||
self._known_events = set(p.name for p in files)
|
||||
|
||||
def show_selected(self):
|
||||
items = self.event_list.selectedItems()
|
||||
if not items:
|
||||
return
|
||||
path = items[0].data(Qt.UserRole)
|
||||
try:
|
||||
self.detail.setPlainText(Path(path).read_text())
|
||||
except Exception as e:
|
||||
self.detail.setPlainText(f"Error: {e}")
|
||||
|
||||
def clear_snapshots(self):
|
||||
if not LOG_DIR.exists():
|
||||
return
|
||||
from PySide6.QtWidgets import QMessageBox
|
||||
if QMessageBox.question(self, "Delete", "Delete all captured snapshots?") != QMessageBox.Yes:
|
||||
return
|
||||
for p in LOG_DIR.glob("snap-*.log"):
|
||||
try:
|
||||
subprocess.run(["pkexec", "rm", str(p)])
|
||||
except Exception:
|
||||
pass
|
||||
self.load_events()
|
||||
self.detail.clear()
|
||||
|
||||
|
||||
def main():
|
||||
app = QApplication(sys.argv)
|
||||
app.setQuitOnLastWindowClosed(False)
|
||||
w = FreezeMonitor()
|
||||
w.show()
|
||||
sys.exit(app.exec())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
99
src/freeze-watcher.sh
Executable file
99
src/freeze-watcher.sh
Executable file
|
|
@ -0,0 +1,99 @@
|
|||
#!/bin/bash
|
||||
# Continuously samples system state. Writes a detailed snapshot when
|
||||
# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
|
||||
|
||||
LOG_DIR=/var/log/freeze-watcher
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# Thresholds
|
||||
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||
DSTATE_THRESHOLD=5 # processes in D state
|
||||
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||
LOAD_THRESHOLD=20 # 1-min load average
|
||||
COOLDOWN=30 # seconds between captures
|
||||
SAMPLE_INTERVAL=1 # seconds between samples
|
||||
|
||||
last_capture=0
|
||||
|
||||
capture_snapshot() {
|
||||
local reason="$1"
|
||||
local now=$(date +%s)
|
||||
local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||
{
|
||||
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||
echo "Time: $(date -Iseconds)"
|
||||
echo
|
||||
echo "## /proc/loadavg"
|
||||
cat /proc/loadavg
|
||||
echo
|
||||
echo "## /proc/stat (procs_running, procs_blocked)"
|
||||
grep -E "procs_running|procs_blocked|intr|ctxt" /proc/stat | head -5
|
||||
echo
|
||||
echo "## PSI cpu/io/memory"
|
||||
for k in cpu io memory; do
|
||||
echo "[$k]"; cat /proc/pressure/$k 2>/dev/null
|
||||
done
|
||||
echo
|
||||
echo "## D-state processes (with kernel wchan = where they're stuck)"
|
||||
ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
|
||||
echo
|
||||
echo "## Top 15 by CPU"
|
||||
ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15
|
||||
echo
|
||||
echo "## Top 15 by RSS memory"
|
||||
ps -eo pid,user,pcpu,pmem,rss,comm --no-headers --sort=-rss | head -15
|
||||
echo
|
||||
echo "## /proc/meminfo summary"
|
||||
grep -E "MemTotal|MemFree|MemAvailable|Dirty|Writeback|Slab|Cached|Buffers" /proc/meminfo
|
||||
echo
|
||||
echo "## iostat (1s snapshot)"
|
||||
iostat -xt 1 2 2>&1 | tail -50
|
||||
echo
|
||||
echo "## interrupts (top by total)"
|
||||
awk 'NR==1{next} {sum=0; for(i=2;i<=NF-2;i++) sum+=$i; if(sum>1000) print sum, $0}' /proc/interrupts | sort -rn | head -15
|
||||
echo
|
||||
echo "## softirq"
|
||||
cat /proc/softirqs
|
||||
echo
|
||||
echo "## Recent dmesg (last 50 lines)"
|
||||
dmesg -T | tail -50
|
||||
echo
|
||||
echo "## journal warnings/errors last 60s"
|
||||
journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
|
||||
} > "$file" 2>&1
|
||||
echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||
}
|
||||
|
||||
echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
|
||||
|
||||
while true; do
|
||||
now=$(date +%s)
|
||||
|
||||
# PSI pressure - parse "some avg10=X.XX"
|
||||
psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
|
||||
psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
|
||||
|
||||
# D-state count
|
||||
dstate=$(ps -eo state --no-headers | grep -c "^D")
|
||||
|
||||
# /proc/stat blocked
|
||||
blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat)
|
||||
|
||||
# Load average
|
||||
load1=$(awk '{print int($1+0.5)}' /proc/loadavg)
|
||||
|
||||
triggered=""
|
||||
[ "${psi_io:-0}" -ge "$PSI_IO_THRESHOLD" ] && triggered="psi_io=$psi_io%"
|
||||
[ "${psi_cpu:-0}" -ge "$PSI_CPU_THRESHOLD" ] && triggered="${triggered:+$triggered, }psi_cpu=$psi_cpu%"
|
||||
[ "${dstate:-0}" -ge "$DSTATE_THRESHOLD" ] && triggered="${triggered:+$triggered, }dstate=$dstate"
|
||||
[ "${blocked:-0}" -ge "$BLOCKED_THRESHOLD" ] && triggered="${triggered:+$triggered, }blocked=$blocked"
|
||||
[ "${load1:-0}" -ge "$LOAD_THRESHOLD" ] && triggered="${triggered:+$triggered, }load1=$load1"
|
||||
|
||||
if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then
|
||||
capture_snapshot "$triggered" &
|
||||
last_capture=$now
|
||||
fi
|
||||
|
||||
sleep "$SAMPLE_INTERVAL"
|
||||
done
|
||||
15
systemd/freeze-watcher.service
Normal file
15
systemd/freeze-watcher.service
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[Unit]
|
||||
Description=Freeze Watcher - captures system state during anomalous load
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/freeze-watcher.sh
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
Nice=-5
|
||||
IOSchedulingClass=realtime
|
||||
IOSchedulingPriority=0
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Loading…
Add table
Add a link
Reference in a new issue