Initial commit: freeze-watcher daemon, GUI, systemd unit, PKGBUILD
This commit is contained in:
commit
d41f861802
10 changed files with 655 additions and 0 deletions
14
.gitignore
vendored
Normal file
14
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Build artifacts
|
||||||
|
*.pkg.tar.*
|
||||||
|
*.tar.gz
|
||||||
|
src/pkg/
|
||||||
|
src/build/
|
||||||
|
|
||||||
|
# Editor
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# Python
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 luna
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
39
Makefile
Normal file
39
Makefile
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
PREFIX ?= /usr/local
|
||||||
|
BINDIR := $(PREFIX)/bin
|
||||||
|
APPDIR := $(PREFIX)/share/applications
|
||||||
|
SYSTEMDDIR := /etc/systemd/system
|
||||||
|
LOGDIR := /var/log/freeze-watcher
|
||||||
|
LOG_GROUP ?= $(shell id -gn)
|
||||||
|
|
||||||
|
.PHONY: install uninstall enable disable status logs clean
|
||||||
|
|
||||||
|
install:
|
||||||
|
install -Dm755 src/freeze-watcher.sh $(DESTDIR)$(BINDIR)/freeze-watcher.sh
|
||||||
|
install -Dm755 src/freeze-monitor-gui $(DESTDIR)$(BINDIR)/freeze-monitor-gui
|
||||||
|
install -Dm644 systemd/freeze-watcher.service $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
|
||||||
|
install -Dm644 desktop/freeze-monitor.desktop $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
|
||||||
|
install -dm775 -g $(LOG_GROUP) $(DESTDIR)$(LOGDIR)
|
||||||
|
@echo "Installed. Run 'make enable' to start the service."
|
||||||
|
|
||||||
|
uninstall:
|
||||||
|
rm -f $(DESTDIR)$(BINDIR)/freeze-watcher.sh
|
||||||
|
rm -f $(DESTDIR)$(BINDIR)/freeze-monitor-gui
|
||||||
|
rm -f $(DESTDIR)$(SYSTEMDDIR)/freeze-watcher.service
|
||||||
|
rm -f $(DESTDIR)$(APPDIR)/freeze-monitor.desktop
|
||||||
|
@echo "Uninstalled. Logs at $(LOGDIR) preserved."
|
||||||
|
|
||||||
|
enable:
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable --now freeze-watcher.service
|
||||||
|
|
||||||
|
disable:
|
||||||
|
systemctl disable --now freeze-watcher.service
|
||||||
|
|
||||||
|
status:
|
||||||
|
systemctl status freeze-watcher.service --no-pager
|
||||||
|
|
||||||
|
logs:
|
||||||
|
journalctl -u freeze-watcher.service -f
|
||||||
|
|
||||||
|
clean:
|
||||||
|
@echo "Nothing to clean (no build artifacts)."
|
||||||
118
README.md
Normal file
118
README.md
Normal file
|
|
@ -0,0 +1,118 @@
|
||||||
|
# freeze-watcher
|
||||||
|
|
||||||
|
A systemd daemon that continuously samples Linux system state and dumps a detailed forensic snapshot whenever freeze-like conditions are detected (D-state spikes, PSI pressure, blocked processes, load avg). Includes a Qt GUI for live metrics and snapshot browsing.
|
||||||
|
|
||||||
|
Built to diagnose intermittent system freezes that don't show up in normal resource monitors — specifically the kind where some apps stall but the mouse keeps moving.
|
||||||
|
|
||||||
|
## What it captures
|
||||||
|
|
||||||
|
Each snapshot is a single text file in `/var/log/freeze-watcher/snap-YYYYMMDD-HHMMSS.log` containing:
|
||||||
|
|
||||||
|
- `loadavg`, `procs_running`, `procs_blocked`, `intr`, `ctxt`
|
||||||
|
- PSI cpu / io / memory pressure (`some` and `full` averages)
|
||||||
|
- D-state processes with kernel `wchan` (where exactly they're stuck)
|
||||||
|
- Top 15 by CPU and RSS
|
||||||
|
- `meminfo` summary
|
||||||
|
- `iostat -xt` snapshot
|
||||||
|
- Top interrupt sources by total + per-CPU breakdown
|
||||||
|
- Full `softirq` table
|
||||||
|
- Last 50 dmesg lines
|
||||||
|
- Last 60s of journal warnings/errors
|
||||||
|
|
||||||
|
The wchan column is the key one — that's how you tell `btrfs_lock_root_node` lock contention from `usb_sg_wait` USB stalls from `folio_wait_bit_common` page-fault stalls.
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
| Path | What |
|
||||||
|
|---|---|
|
||||||
|
| `src/freeze-watcher.sh` | The daemon — samples every 1s, captures on threshold |
|
||||||
|
| `src/freeze-monitor-gui` | PySide6 GUI: live metrics + snapshot browser + tray icon |
|
||||||
|
| `systemd/freeze-watcher.service` | systemd unit |
|
||||||
|
| `desktop/freeze-monitor.desktop` | App launcher entry |
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
### Via Makefile (system-wide install to `/usr/local`)
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo make install
|
||||||
|
sudo make enable
|
||||||
|
```
|
||||||
|
|
||||||
|
### As an Arch package
|
||||||
|
|
||||||
|
```
|
||||||
|
./packaging/build-package.sh
|
||||||
|
sudo pacman -U freeze-watcher-*.pkg.tar.zst
|
||||||
|
sudo systemctl enable --now freeze-watcher.service
|
||||||
|
```
|
||||||
|
|
||||||
|
## Update workflow
|
||||||
|
|
||||||
|
When you change something:
|
||||||
|
|
||||||
|
```
|
||||||
|
# 1. Edit files in src/ or systemd/
|
||||||
|
# 2. Reinstall
|
||||||
|
sudo make install
|
||||||
|
|
||||||
|
# 3. If the daemon script changed, restart it
|
||||||
|
sudo systemctl restart freeze-watcher.service
|
||||||
|
|
||||||
|
# 4. If the GUI changed, just relaunch it (it's a regular Qt app)
|
||||||
|
```
|
||||||
|
|
||||||
|
If you're using the Arch package: `./packaging/build-package.sh && sudo pacman -U *.pkg.tar.zst` will rebuild and upgrade in place.
|
||||||
|
|
||||||
|
## Tuning thresholds
|
||||||
|
|
||||||
|
Open `src/freeze-watcher.sh` and edit the constants at the top:
|
||||||
|
|
||||||
|
```
|
||||||
|
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||||
|
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||||
|
DSTATE_THRESHOLD=5 # processes in D state
|
||||||
|
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||||
|
LOAD_THRESHOLD=20 # 1-min load average
|
||||||
|
COOLDOWN=30 # seconds between captures
|
||||||
|
SAMPLE_INTERVAL=1 # seconds between samples
|
||||||
|
```
|
||||||
|
|
||||||
|
## GUI
|
||||||
|
|
||||||
|
Launch from app menu (search "Freeze Watcher") or `freeze-monitor-gui`.
|
||||||
|
|
||||||
|
- Live PSI / D-state / load metrics with green/orange/red color coding (1s refresh)
|
||||||
|
- Service status with start/stop buttons (uses `pkexec`)
|
||||||
|
- Captured event list — click any row for full snapshot view
|
||||||
|
- Closes to system tray; quit from the tray menu
|
||||||
|
- Notification when a new snapshot is captured
|
||||||
|
|
||||||
|
## Reading a snapshot
|
||||||
|
|
||||||
|
The most informative section is **D-state processes** — the `wchan` tells you what kernel function each process is blocked in. Common ones:
|
||||||
|
|
||||||
|
| wchan | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `btrfs_lock_root_node` | btrfs metadata lock contention (often qgroups + snapshots) |
|
||||||
|
| `btrfs_search_slot` | btrfs tree walk waiting for lock |
|
||||||
|
| `btrfs_sync_log` | fsync waiting for transaction commit |
|
||||||
|
| `folio_wait_bit_common` | Waiting for memory page (often swap-in) |
|
||||||
|
| `usb_sg_wait` | USB device stall |
|
||||||
|
| `io_schedule` | Generic I/O wait |
|
||||||
|
|
||||||
|
Cross-reference `wchan` with PSI io and PSI cpu to distinguish lock contention (low PSI) from genuine resource pressure (high PSI).
|
||||||
|
|
||||||
|
## Future ideas
|
||||||
|
|
||||||
|
- Per-cgroup PSI capture
|
||||||
|
- eBPF tracing for long-running syscalls
|
||||||
|
- Web UI option (no Qt requirement)
|
||||||
|
- Auto-prune old snapshots
|
||||||
|
- Configurable `/etc/freeze-watcher.conf` instead of inline thresholds
|
||||||
|
- Integration with desktop notifications via `notify-send`
|
||||||
|
- Optional perf-profile capture during event
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT
|
||||||
9
desktop/freeze-monitor.desktop
Normal file
9
desktop/freeze-monitor.desktop
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
[Desktop Entry]
|
||||||
|
Type=Application
|
||||||
|
Name=Freeze Watcher
|
||||||
|
Comment=Monitor system freezes and view captured snapshots
|
||||||
|
Exec=/usr/local/bin/freeze-monitor-gui
|
||||||
|
Icon=utilities-system-monitor
|
||||||
|
Terminal=false
|
||||||
|
Categories=System;Monitor;
|
||||||
|
StartupNotify=true
|
||||||
39
packaging/PKGBUILD
Normal file
39
packaging/PKGBUILD
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
# Maintainer: luna <anassaeneroi@pm.me>
|
||||||
|
pkgname=freeze-watcher
|
||||||
|
pkgver=0.1.0
|
||||||
|
pkgrel=1
|
||||||
|
pkgdesc="Continuously samples system state and snapshots detailed diagnostics when freeze conditions are detected"
|
||||||
|
arch=('any')
|
||||||
|
url="https://github.com/luna/freeze-watcher"
|
||||||
|
license=('MIT')
|
||||||
|
depends=('bash' 'systemd' 'python' 'pyside6' 'sysstat' 'procps-ng' 'util-linux')
|
||||||
|
optdepends=('xdg-utils: open log directory from GUI')
|
||||||
|
source=("$pkgname-$pkgver.tar.gz::file://$PWD/../$pkgname-$pkgver.tar.gz")
|
||||||
|
sha256sums=('SKIP')
|
||||||
|
backup=('etc/systemd/system/freeze-watcher.service')
|
||||||
|
|
||||||
|
package() {
|
||||||
|
cd "$srcdir/$pkgname-$pkgver"
|
||||||
|
make DESTDIR="$pkgdir" PREFIX=/usr install
|
||||||
|
}
|
||||||
|
|
||||||
|
post_install() {
|
||||||
|
echo "==> Run 'sudo systemctl enable --now freeze-watcher.service' to start"
|
||||||
|
echo "==> Snapshots will be written to /var/log/freeze-watcher/"
|
||||||
|
echo "==> Launch the GUI from your app menu (Freeze Watcher) or 'freeze-monitor-gui'"
|
||||||
|
}
|
||||||
|
|
||||||
|
post_upgrade() {
|
||||||
|
systemctl daemon-reload
|
||||||
|
if systemctl is-active --quiet freeze-watcher.service; then
|
||||||
|
echo "==> Restart the service to pick up changes:"
|
||||||
|
echo " sudo systemctl restart freeze-watcher.service"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
post_remove() {
|
||||||
|
if systemctl is-enabled --quiet freeze-watcher.service 2>/dev/null; then
|
||||||
|
systemctl disable --now freeze-watcher.service
|
||||||
|
fi
|
||||||
|
echo "==> Logs at /var/log/freeze-watcher/ preserved. Remove manually if desired."
|
||||||
|
}
|
||||||
26
packaging/build-package.sh
Executable file
26
packaging/build-package.sh
Executable file
|
|
@ -0,0 +1,26 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Build a local Arch package from the current source tree.
|
||||||
|
# Usage: ./packaging/build-package.sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
PKGNAME=freeze-watcher
|
||||||
|
PKGVER=$(awk -F= '/^pkgver=/{print $2}' "$REPO_ROOT/packaging/PKGBUILD")
|
||||||
|
|
||||||
|
cd "$REPO_ROOT"
|
||||||
|
tmp=$(mktemp -d)
|
||||||
|
trap "rm -rf $tmp" EXIT
|
||||||
|
|
||||||
|
# Stage source as a tarball (PKGBUILD expects it as $pkgname-$pkgver.tar.gz)
|
||||||
|
mkdir "$tmp/$PKGNAME-$PKGVER"
|
||||||
|
cp -r src systemd desktop Makefile README.md LICENSE "$tmp/$PKGNAME-$PKGVER/" 2>/dev/null || true
|
||||||
|
tar czf "$tmp/$PKGNAME-$PKGVER.tar.gz" -C "$tmp" "$PKGNAME-$PKGVER"
|
||||||
|
|
||||||
|
cp packaging/PKGBUILD "$tmp/PKGBUILD"
|
||||||
|
cd "$tmp"
|
||||||
|
makepkg -f --noconfirm
|
||||||
|
mv "$PKGNAME-$PKGVER-"*.pkg.tar.* "$REPO_ROOT/"
|
||||||
|
echo
|
||||||
|
echo "Built: $REPO_ROOT/$(ls "$REPO_ROOT" | grep "^$PKGNAME-$PKGVER" | head -1)"
|
||||||
|
echo "Install with: sudo pacman -U <path-to-pkg>"
|
||||||
275
src/freeze-monitor-gui
Executable file
275
src/freeze-monitor-gui
Executable file
|
|
@ -0,0 +1,275 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""GUI for freeze-watcher service. Shows live metrics and captured snapshots."""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from PySide6.QtCore import Qt, QTimer
|
||||||
|
from PySide6.QtGui import QFont, QColor, QAction, QIcon
|
||||||
|
from PySide6.QtWidgets import (
|
||||||
|
QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
|
||||||
|
QLabel, QListWidget, QListWidgetItem, QTextEdit, QPushButton,
|
||||||
|
QSplitter, QGroupBox, QGridLayout, QSystemTrayIcon, QMenu, QStyle
|
||||||
|
)
|
||||||
|
|
||||||
|
LOG_DIR = Path("/var/log/freeze-watcher")
|
||||||
|
SERVICE = "freeze-watcher.service"
|
||||||
|
|
||||||
|
|
||||||
|
def read_psi(name):
|
||||||
|
try:
|
||||||
|
line = (Path("/proc/pressure") / name).read_text().splitlines()[0]
|
||||||
|
m = re.search(r"avg10=([\d.]+)", line)
|
||||||
|
return float(m.group(1)) if m else 0.0
|
||||||
|
except Exception:
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def read_dstate_count():
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
["ps", "-eo", "state", "--no-headers"],
|
||||||
|
capture_output=True, text=True, timeout=2,
|
||||||
|
).stdout
|
||||||
|
return sum(1 for ln in out.splitlines() if ln.startswith("D"))
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def read_loadavg():
|
||||||
|
try:
|
||||||
|
return float(Path("/proc/loadavg").read_text().split()[0])
|
||||||
|
except Exception:
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def read_procs_blocked():
|
||||||
|
try:
|
||||||
|
for ln in Path("/proc/stat").read_text().splitlines():
|
||||||
|
if ln.startswith("procs_blocked"):
|
||||||
|
return int(ln.split()[1])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def service_active():
|
||||||
|
r = subprocess.run(["systemctl", "is-active", SERVICE], capture_output=True, text=True)
|
||||||
|
return r.stdout.strip() == "active"
|
||||||
|
|
||||||
|
|
||||||
|
class MetricLabel(QLabel):
|
||||||
|
def __init__(self, name):
|
||||||
|
super().__init__("--")
|
||||||
|
self.name = name
|
||||||
|
self.setAlignment(Qt.AlignCenter)
|
||||||
|
self.setMinimumHeight(48)
|
||||||
|
f = QFont()
|
||||||
|
f.setPointSize(18)
|
||||||
|
f.setBold(True)
|
||||||
|
self.setFont(f)
|
||||||
|
self._set_color("#888")
|
||||||
|
|
||||||
|
def _set_color(self, hex_):
|
||||||
|
self.setStyleSheet(
|
||||||
|
f"background:#1e1e1e; color:{hex_}; border:1px solid #333; border-radius:6px; padding:6px;"
|
||||||
|
)
|
||||||
|
|
||||||
|
def set_value(self, value, warn, crit, suffix=""):
|
||||||
|
self.setText(f"{value}{suffix}")
|
||||||
|
if value >= crit:
|
||||||
|
self._set_color("#ff5555")
|
||||||
|
elif value >= warn:
|
||||||
|
self._set_color("#f5a623")
|
||||||
|
else:
|
||||||
|
self._set_color("#5cdb95")
|
||||||
|
|
||||||
|
|
||||||
|
class FreezeMonitor(QMainWindow):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.setWindowTitle("Freeze Watcher")
|
||||||
|
self.resize(1100, 700)
|
||||||
|
|
||||||
|
central = QWidget()
|
||||||
|
self.setCentralWidget(central)
|
||||||
|
root = QVBoxLayout(central)
|
||||||
|
|
||||||
|
# Status bar
|
||||||
|
self.status_label = QLabel()
|
||||||
|
self.status_label.setStyleSheet("padding:8px; font-weight:bold;")
|
||||||
|
root.addWidget(self.status_label)
|
||||||
|
|
||||||
|
# Live metrics grid
|
||||||
|
metrics_box = QGroupBox("Live metrics (1s refresh)")
|
||||||
|
grid = QGridLayout(metrics_box)
|
||||||
|
labels = ["PSI IO %", "PSI CPU %", "PSI Mem %", "D-state", "Blocked", "Load (1m)"]
|
||||||
|
self.metrics = {}
|
||||||
|
for i, name in enumerate(labels):
|
||||||
|
grid.addWidget(QLabel(name, alignment=Qt.AlignCenter), 0, i)
|
||||||
|
m = MetricLabel(name)
|
||||||
|
self.metrics[name] = m
|
||||||
|
grid.addWidget(m, 1, i)
|
||||||
|
root.addWidget(metrics_box)
|
||||||
|
|
||||||
|
# Buttons
|
||||||
|
btn_row = QHBoxLayout()
|
||||||
|
self.btn_start = QPushButton("Start service")
|
||||||
|
self.btn_stop = QPushButton("Stop service")
|
||||||
|
self.btn_refresh = QPushButton("Refresh events")
|
||||||
|
self.btn_open = QPushButton("Open log dir")
|
||||||
|
self.btn_clear = QPushButton("Delete all snapshots")
|
||||||
|
for b in (self.btn_start, self.btn_stop, self.btn_refresh, self.btn_open, self.btn_clear):
|
||||||
|
btn_row.addWidget(b)
|
||||||
|
btn_row.addStretch()
|
||||||
|
self.btn_start.clicked.connect(lambda: self._svc("start"))
|
||||||
|
self.btn_stop.clicked.connect(lambda: self._svc("stop"))
|
||||||
|
self.btn_refresh.clicked.connect(self.load_events)
|
||||||
|
self.btn_open.clicked.connect(lambda: subprocess.Popen(["xdg-open", str(LOG_DIR)]))
|
||||||
|
self.btn_clear.clicked.connect(self.clear_snapshots)
|
||||||
|
root.addLayout(btn_row)
|
||||||
|
|
||||||
|
# Splitter: list | detail
|
||||||
|
splitter = QSplitter(Qt.Horizontal)
|
||||||
|
self.event_list = QListWidget()
|
||||||
|
self.event_list.itemSelectionChanged.connect(self.show_selected)
|
||||||
|
splitter.addWidget(self.event_list)
|
||||||
|
self.detail = QTextEdit()
|
||||||
|
self.detail.setReadOnly(True)
|
||||||
|
self.detail.setFont(QFont("monospace"))
|
||||||
|
splitter.addWidget(self.detail)
|
||||||
|
splitter.setSizes([350, 750])
|
||||||
|
root.addWidget(splitter, stretch=1)
|
||||||
|
|
||||||
|
self._known_events = set()
|
||||||
|
|
||||||
|
# Tray (created before timer so tick() can use it)
|
||||||
|
self.tray = QSystemTrayIcon(self.style().standardIcon(QStyle.SP_ComputerIcon), self)
|
||||||
|
menu = QMenu()
|
||||||
|
show_act = QAction("Show window", self)
|
||||||
|
show_act.triggered.connect(self.showNormal)
|
||||||
|
quit_act = QAction("Quit", self)
|
||||||
|
quit_act.triggered.connect(QApplication.quit)
|
||||||
|
menu.addAction(show_act)
|
||||||
|
menu.addAction(quit_act)
|
||||||
|
self.tray.setContextMenu(menu)
|
||||||
|
self.tray.activated.connect(self._tray_clicked)
|
||||||
|
self.tray.show()
|
||||||
|
|
||||||
|
# Timer
|
||||||
|
self.timer = QTimer()
|
||||||
|
self.timer.timeout.connect(self.tick)
|
||||||
|
self.timer.start(1000)
|
||||||
|
self.load_events()
|
||||||
|
self.tick()
|
||||||
|
|
||||||
|
def _tray_clicked(self, reason):
|
||||||
|
if reason == QSystemTrayIcon.Trigger:
|
||||||
|
if self.isVisible():
|
||||||
|
self.hide()
|
||||||
|
else:
|
||||||
|
self.showNormal()
|
||||||
|
self.raise_()
|
||||||
|
|
||||||
|
def closeEvent(self, ev):
|
||||||
|
ev.ignore()
|
||||||
|
self.hide()
|
||||||
|
|
||||||
|
def _svc(self, action):
|
||||||
|
subprocess.run(["pkexec", "systemctl", action, SERVICE])
|
||||||
|
self.tick()
|
||||||
|
|
||||||
|
def tick(self):
|
||||||
|
psi_io = read_psi("io")
|
||||||
|
psi_cpu = read_psi("cpu")
|
||||||
|
psi_mem = read_psi("memory")
|
||||||
|
dstate = read_dstate_count()
|
||||||
|
blocked = read_procs_blocked()
|
||||||
|
load = read_loadavg()
|
||||||
|
|
||||||
|
self.metrics["PSI IO %"].set_value(int(psi_io), 30, 60, "")
|
||||||
|
self.metrics["PSI CPU %"].set_value(int(psi_cpu), 30, 60, "")
|
||||||
|
self.metrics["PSI Mem %"].set_value(int(psi_mem), 20, 50, "")
|
||||||
|
self.metrics["D-state"].set_value(dstate, 3, 6, "")
|
||||||
|
self.metrics["Blocked"].set_value(blocked, 3, 6, "")
|
||||||
|
self.metrics["Load (1m)"].set_value(int(load), 8, 16, "")
|
||||||
|
|
||||||
|
if service_active():
|
||||||
|
self.status_label.setText("● Service: ACTIVE")
|
||||||
|
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#5cdb95;")
|
||||||
|
else:
|
||||||
|
self.status_label.setText("● Service: STOPPED")
|
||||||
|
self.status_label.setStyleSheet("padding:8px; font-weight:bold; color:#ff5555;")
|
||||||
|
|
||||||
|
# Auto-refresh events list when a new file appears
|
||||||
|
current = set(p.name for p in LOG_DIR.glob("snap-*.log")) if LOG_DIR.exists() else set()
|
||||||
|
if current != self._known_events:
|
||||||
|
new = current - self._known_events
|
||||||
|
self._known_events = current
|
||||||
|
self.load_events()
|
||||||
|
for n in new:
|
||||||
|
self.tray.showMessage(
|
||||||
|
"Freeze Watcher",
|
||||||
|
f"Captured: {n}",
|
||||||
|
QSystemTrayIcon.Warning,
|
||||||
|
5000,
|
||||||
|
)
|
||||||
|
|
||||||
|
def load_events(self):
|
||||||
|
self.event_list.clear()
|
||||||
|
if not LOG_DIR.exists():
|
||||||
|
return
|
||||||
|
files = sorted(LOG_DIR.glob("snap-*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||||
|
for p in files:
|
||||||
|
reason = ""
|
||||||
|
try:
|
||||||
|
first = p.read_text().splitlines()[0]
|
||||||
|
if "TRIGGERED:" in first:
|
||||||
|
reason = first.split("TRIGGERED:", 1)[1].strip().rstrip("=").strip()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
ts = datetime.fromtimestamp(p.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
item = QListWidgetItem(f"{ts}\n {reason}")
|
||||||
|
item.setData(Qt.UserRole, str(p))
|
||||||
|
self.event_list.addItem(item)
|
||||||
|
self._known_events = set(p.name for p in files)
|
||||||
|
|
||||||
|
def show_selected(self):
|
||||||
|
items = self.event_list.selectedItems()
|
||||||
|
if not items:
|
||||||
|
return
|
||||||
|
path = items[0].data(Qt.UserRole)
|
||||||
|
try:
|
||||||
|
self.detail.setPlainText(Path(path).read_text())
|
||||||
|
except Exception as e:
|
||||||
|
self.detail.setPlainText(f"Error: {e}")
|
||||||
|
|
||||||
|
def clear_snapshots(self):
|
||||||
|
if not LOG_DIR.exists():
|
||||||
|
return
|
||||||
|
from PySide6.QtWidgets import QMessageBox
|
||||||
|
if QMessageBox.question(self, "Delete", "Delete all captured snapshots?") != QMessageBox.Yes:
|
||||||
|
return
|
||||||
|
for p in LOG_DIR.glob("snap-*.log"):
|
||||||
|
try:
|
||||||
|
subprocess.run(["pkexec", "rm", str(p)])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self.load_events()
|
||||||
|
self.detail.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
app = QApplication(sys.argv)
|
||||||
|
app.setQuitOnLastWindowClosed(False)
|
||||||
|
w = FreezeMonitor()
|
||||||
|
w.show()
|
||||||
|
sys.exit(app.exec())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
99
src/freeze-watcher.sh
Executable file
99
src/freeze-watcher.sh
Executable file
|
|
@ -0,0 +1,99 @@
|
||||||
|
#!/bin/bash
|
||||||
|
# Continuously samples system state. Writes a detailed snapshot when
|
||||||
|
# anomaly thresholds are crossed (D-states, PSI pressure, blocked procs).
|
||||||
|
|
||||||
|
LOG_DIR=/var/log/freeze-watcher
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
|
||||||
|
# Thresholds
|
||||||
|
PSI_IO_THRESHOLD=40 # io pressure avg10 % to trigger
|
||||||
|
PSI_CPU_THRESHOLD=50 # cpu pressure avg10 % to trigger
|
||||||
|
DSTATE_THRESHOLD=5 # processes in D state
|
||||||
|
BLOCKED_THRESHOLD=5 # procs_blocked from /proc/stat
|
||||||
|
LOAD_THRESHOLD=20 # 1-min load average
|
||||||
|
COOLDOWN=30 # seconds between captures
|
||||||
|
SAMPLE_INTERVAL=1 # seconds between samples
|
||||||
|
|
||||||
|
last_capture=0
|
||||||
|
|
||||||
|
capture_snapshot() {
|
||||||
|
local reason="$1"
|
||||||
|
local now=$(date +%s)
|
||||||
|
local file="$LOG_DIR/snap-$(date +%Y%m%d-%H%M%S).log"
|
||||||
|
{
|
||||||
|
echo "=== FREEZE EVENT TRIGGERED: $reason ==="
|
||||||
|
echo "Time: $(date -Iseconds)"
|
||||||
|
echo
|
||||||
|
echo "## /proc/loadavg"
|
||||||
|
cat /proc/loadavg
|
||||||
|
echo
|
||||||
|
echo "## /proc/stat (procs_running, procs_blocked)"
|
||||||
|
grep -E "procs_running|procs_blocked|intr|ctxt" /proc/stat | head -5
|
||||||
|
echo
|
||||||
|
echo "## PSI cpu/io/memory"
|
||||||
|
for k in cpu io memory; do
|
||||||
|
echo "[$k]"; cat /proc/pressure/$k 2>/dev/null
|
||||||
|
done
|
||||||
|
echo
|
||||||
|
echo "## D-state processes (with kernel wchan = where they're stuck)"
|
||||||
|
ps -eo state,pid,user,wchan:25,comm,args --no-headers | awk '$1=="D"' | head -30
|
||||||
|
echo
|
||||||
|
echo "## Top 15 by CPU"
|
||||||
|
ps -eo pid,user,pcpu,pmem,state,comm --no-headers --sort=-pcpu | head -15
|
||||||
|
echo
|
||||||
|
echo "## Top 15 by RSS memory"
|
||||||
|
ps -eo pid,user,pcpu,pmem,rss,comm --no-headers --sort=-rss | head -15
|
||||||
|
echo
|
||||||
|
echo "## /proc/meminfo summary"
|
||||||
|
grep -E "MemTotal|MemFree|MemAvailable|Dirty|Writeback|Slab|Cached|Buffers" /proc/meminfo
|
||||||
|
echo
|
||||||
|
echo "## iostat (1s snapshot)"
|
||||||
|
iostat -xt 1 2 2>&1 | tail -50
|
||||||
|
echo
|
||||||
|
echo "## interrupts (top by total)"
|
||||||
|
awk 'NR==1{next} {sum=0; for(i=2;i<=NF-2;i++) sum+=$i; if(sum>1000) print sum, $0}' /proc/interrupts | sort -rn | head -15
|
||||||
|
echo
|
||||||
|
echo "## softirq"
|
||||||
|
cat /proc/softirqs
|
||||||
|
echo
|
||||||
|
echo "## Recent dmesg (last 50 lines)"
|
||||||
|
dmesg -T | tail -50
|
||||||
|
echo
|
||||||
|
echo "## journal warnings/errors last 60s"
|
||||||
|
journalctl --since "60 seconds ago" -p warning --no-pager 2>&1 | tail -40
|
||||||
|
} > "$file" 2>&1
|
||||||
|
echo "$(date -Iseconds) Captured snapshot to $file (reason: $reason)" >> "$LOG_DIR/events.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "$(date -Iseconds) freeze-watcher started (pid $$)" >> "$LOG_DIR/events.log"
|
||||||
|
|
||||||
|
while true; do
|
||||||
|
now=$(date +%s)
|
||||||
|
|
||||||
|
# PSI pressure - parse "some avg10=X.XX"
|
||||||
|
psi_io=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/io 2>/dev/null)
|
||||||
|
psi_cpu=$(awk '/^some/{for(i=1;i<=NF;i++) if($i~/^avg10=/){gsub(/avg10=/,"",$i); print int($i+0.5); exit}}' /proc/pressure/cpu 2>/dev/null)
|
||||||
|
|
||||||
|
# D-state count
|
||||||
|
dstate=$(ps -eo state --no-headers | grep -c "^D")
|
||||||
|
|
||||||
|
# /proc/stat blocked
|
||||||
|
blocked=$(awk '/^procs_blocked/{print $2}' /proc/stat)
|
||||||
|
|
||||||
|
# Load average
|
||||||
|
load1=$(awk '{print int($1+0.5)}' /proc/loadavg)
|
||||||
|
|
||||||
|
triggered=""
|
||||||
|
[ "${psi_io:-0}" -ge "$PSI_IO_THRESHOLD" ] && triggered="psi_io=$psi_io%"
|
||||||
|
[ "${psi_cpu:-0}" -ge "$PSI_CPU_THRESHOLD" ] && triggered="${triggered:+$triggered, }psi_cpu=$psi_cpu%"
|
||||||
|
[ "${dstate:-0}" -ge "$DSTATE_THRESHOLD" ] && triggered="${triggered:+$triggered, }dstate=$dstate"
|
||||||
|
[ "${blocked:-0}" -ge "$BLOCKED_THRESHOLD" ] && triggered="${triggered:+$triggered, }blocked=$blocked"
|
||||||
|
[ "${load1:-0}" -ge "$LOAD_THRESHOLD" ] && triggered="${triggered:+$triggered, }load1=$load1"
|
||||||
|
|
||||||
|
if [ -n "$triggered" ] && [ $((now - last_capture)) -ge "$COOLDOWN" ]; then
|
||||||
|
capture_snapshot "$triggered" &
|
||||||
|
last_capture=$now
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep "$SAMPLE_INTERVAL"
|
||||||
|
done
|
||||||
15
systemd/freeze-watcher.service
Normal file
15
systemd/freeze-watcher.service
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Freeze Watcher - captures system state during anomalous load
|
||||||
|
After=multi-user.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStart=/usr/local/bin/freeze-watcher.sh
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
Nice=-5
|
||||||
|
IOSchedulingClass=realtime
|
||||||
|
IOSchedulingPriority=0
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
Loading…
Add table
Add a link
Reference in a new issue