feat(go): port FIM/pkgdb/rootcheck integrity engines and add go-sidecar dashboard consumer

Adds sidecar-only, --state-dir-gated FIM, package-DB-anchor, and rootcheck
engines to the Go migration sidecar, wired into agent.Sweep/Initialize
behind the existing baseline lifecycle. Adds a read-only, schema-checked
Python dashboard consumer (go_sidecar_state_dir, /api/go-sidecar/*, Sidecar
tab) that never starts, stops, or mutates the Go sidecar's state.

Also fixes drift found while reconciling this work: enodia_sentinel/web.py
had reinvented local schema constants instead of using the canonical
enodia_sentinel/schemas.py catalog (now registers enodia.go.sidecar.v1
there and reuses ALERT_SNAPSHOT_V1/INCIDENT_V1); docs/RULES.md had drifted
from what `rules docs` actually generates for SIDs 100069-100078 (ruleops.py
was missing metadata for four SIDs and had stale drill text for four more),
now back in sync with a regression test pinning them together.

Updates CLAUDE.md, README.md, go-agent/README.md, and docs/{ROADMAP,
GO_PORT_HANDOFF,SURICATA_ASSIMILATION,THREAT_MODEL,OPERATIONS,SCHEMAS,
COMMAND_REFERENCE}.md to reflect the landed work.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NQivSKBqQJsayz1xcYqWzZ
This commit is contained in:
Luna 2026-07-24 09:59:09 -07:00
parent 1d1dee7f6e
commit d835386381
No known key found for this signature in database
43 changed files with 3419 additions and 72 deletions

View file

@ -173,6 +173,11 @@ class Config:
web_token: str = "" # bearer token; auto-generated if empty + non-local
web_tls_cert: str = "" # "" = auto-generate self-signed cert under log_dir
web_tls_key: str = "" # "" = auto-generate private key under log_dir
# Optional, read-only Go validation-sidecar evidence tree. This must stay
# separate from log_dir so the Python daemon remains the production source
# of truth and the dashboard never mistakes migration evidence for Python
# alert state.
go_sidecar_state_dir: str = ""
# paths
log_dir: Path = Path("/var/log/enodia-sentinel")

View file

@ -58,6 +58,11 @@ def render_markdown(cfg: Config) -> str:
"Use `enodia-sentinel rules list/show/test` to inspect rules and validate "
"event fixtures locally.",
"",
"Built-in event-rule fixtures live in `tests/fixtures/sids/` and can be replayed",
"with `enodia-sentinel rules test tests/fixtures/sids/<sid>-<slug>.json`.",
"Non-event built-in SIDs are covered by safe offline drills in `tests/sid_drills.py`;",
"`python3 -m unittest tests.test_sid_coverage -v` verifies the full registry.",
"",
]
for rule in list_rules(cfg):
doc = _RULE_DOCS.get(rule["sid"], {})
@ -284,33 +289,61 @@ _RULE_DOCS: dict[int, dict[str, Any]] = {
],
"drill": "Use `rules test` with a `listen` event whose `comm` is an interpreter and whose `local_port` is not a common service port.",
},
100069: {
"false_positives": [
"Installer, configuration-management, or recovery scripts that legitimately write persistence paths from an interpreter.",
"Administrative SSH-key or service-unit maintenance performed by a reviewed script.",
],
"drill": "Use `rules test` with `tests/fixtures/sids/100069-interpreter-persistence-write.json`.",
},
100070: {
"false_positives": [
"Package post-install scripts or configuration-management agents adjusting mode/owner on system service, sudoers, cron, or SSH key files.",
"Manual recovery work that repairs permissions after a known-good change.",
],
"drill": "Use `rules test` with `tests/fixtures/sids/100070-persistence-permission-change.json`.",
},
100071: {
"false_positives": [
"Reviewed administrative wrappers that intentionally run interpreters across a root UID transition.",
"Privileged installer or service-management scripts during maintenance windows.",
],
"drill": "Use `rules test` with `tests/fixtures/sids/100071-interpreter-setuid-root.json`.",
},
100072: {
"false_positives": [
"Reviewed administrative wrappers that intentionally run interpreters across a root GID transition.",
"Privileged installer or service-management scripts during maintenance windows.",
],
"drill": "Use `rules test` with `tests/fixtures/sids/100072-interpreter-setgid-root.json`.",
},
100073: {
"false_positives": [
"Developer or diagnostic scripts binding high local ports intentionally.",
"Short-lived local service wrappers that bind before handing sockets to a supervised process.",
],
"drill": "Use `rules test` with a `bind` event whose `comm` is an interpreter and whose `local_port` is not a common service port.",
"drill": "Use `rules test` with `tests/fixtures/sids/100073-interpreter-unusual-bind.json`.",
},
100076: {
"false_positives": [
"Developer servers or test harnesses accepting inbound public traffic from an interpreter.",
"Administrative troubleshooting with temporary netcat/socat listeners.",
],
"drill": "Use `rules test` with an `accept` event whose `comm` is an interpreter, public `peer_ip`, and unusual `local_port`.",
"drill": "Use `rules test` with `tests/fixtures/sids/100076-interpreter-unusual-accept.json`.",
},
100077: {
"false_positives": [
"Privileged maintenance scripts that intentionally adjust capabilities during controlled administration.",
"Container or network lab setup scripts using interpreters to configure namespaces or packet capture.",
],
"drill": "Use `rules test` with a `capset` event containing a sensitive capability such as `CAP_SYS_ADMIN`.",
"drill": "Use `rules test` with `tests/fixtures/sids/100077-interpreter-capset-sensitive.json`.",
},
100078: {
"false_positives": [
"Kernel development labs loading locally built modules from temporary build directories.",
"Driver troubleshooting sessions that intentionally test an unsigned module from a writable path.",
],
"drill": "Use `rules test` with a `module_load` event whose `path` is under `/tmp`, `/var/tmp`, `/dev/shm`, or `/run/user`.",
"drill": "Use `rules test` with `tests/fixtures/sids/100078-module-load-writable-path.json`.",
},
}

View file

@ -19,3 +19,4 @@ RESPONSE_PLAN_V1 = "enodia.response.plan.v1"
RESPONSE_AUDIT_V1 = "enodia.response.audit.v1"
RECONCILE_V1 = "enodia.reconcile.v1"
EVENT_V1 = "enodia.event.v1"
GO_SIDECAR_V1 = "enodia.go.sidecar.v1"

View file

@ -313,6 +313,7 @@
<button data-tab="alerts">Alerts</button>
<button data-tab="posture">Posture</button>
<button data-tab="integrity">Integrity</button>
<button data-tab="sidecar">Sidecar</button>
<button data-tab="rules">Rules</button>
<button data-tab="events">Events</button>
</div>
@ -320,6 +321,7 @@
<section id="alertsTab" hidden></section>
<section id="postureTab" hidden></section>
<section id="integrityTab" hidden></section>
<section id="sidecarTab" hidden></section>
<section id="rulesTab" hidden></section>
<section id="eventsTab" hidden></section>
</main>
@ -333,7 +335,7 @@
<script>
const TOKEN = new URLSearchParams(location.search).get("token") || "";
const HDR = TOKEN ? {Authorization:"Bearer "+TOKEN} : {};
const TABS = new Set(["incident","alerts","posture","integrity","rules","events"]);
const TABS = new Set(["incident","alerts","posture","integrity","sidecar","rules","events"]);
const THEME_OPTIONS = [
["console","Console"],["paper","Paper"],["contrast","Contrast"],
["pride","LGBTQ"],["trans","Trans"],["dracula","Dracula"],
@ -346,7 +348,7 @@ function tabFromHash(){
const name = location.hash.replace(/^#/,"");
return TABS.has(name) ? name : "incident";
}
const state = {incidents:[], alerts:[], posture:null, integrity:null, rules:null, selected:null, incident:null, plan:null, tab:tabFromHash()};
const state = {incidents:[], alerts:[], posture:null, integrity:null, rules:null, sidecar:null, selected:null, incident:null, plan:null, tab:tabFromHash()};
async function api(path){
const r = await fetch(path,{headers:HDR});
@ -398,10 +400,10 @@ function setSettings(open){
async function refresh(){
try{
const [status, incidents, alerts, posture, integrity, rules, events] = await Promise.all([
api("/api/status"), api("/api/incidents"), api("/api/alerts"), api("/api/posture"), api("/api/integrity"), api("/api/rules"), api("/api/events")
const [status, incidents, alerts, posture, integrity, rules, events, sidecar] = await Promise.all([
api("/api/status"), api("/api/incidents"), api("/api/alerts"), api("/api/posture"), api("/api/integrity"), api("/api/rules"), api("/api/events"), loadSidecar()
]);
state.incidents = incidents; state.alerts = alerts; state.posture = posture; state.integrity = integrity; state.rules = rules; state.events = events.events || [];
state.incidents = incidents; state.alerts = alerts; state.posture = posture; state.integrity = integrity; state.rules = rules; state.events = events.events || []; state.sidecar = sidecar;
renderStatus(status); renderLists();
if(!state.selected && incidents.length) await openIncident(incidents[0].id);
else if(state.selected) await openIncident(state.selected, false);
@ -409,6 +411,20 @@ async function refresh(){
}catch(e){ showError(e.message); }
}
async function loadSidecar(){
// The migration evidence tree is optional. A sidecar read failure must not
// hide the Python daemon's authoritative console data or break refresh. Keep
// its requests separate so a future sidecar-only endpoint cannot accidentally
// become a dependency of the production management surface.
try{
const [status, alerts, incidents, events] = await Promise.all([
api("/api/go-sidecar/status"), api("/api/go-sidecar/alerts"),
api("/api/go-sidecar/incidents"), api("/api/go-sidecar/events")
]);
return {status, alerts, incidents, events, error:null};
}catch(e){ return {status:null, alerts:[], incidents:[], events:{events:[]}, error:e.message}; }
}
function renderStatus(st){
document.getElementById("dot").className = "dot "+(st.running && !st.heartbeat_stale ? "up" : "");
let hb = st.heartbeat_age == null ? "no heartbeat" : Math.round(st.heartbeat_age)+"s heartbeat";
@ -473,6 +489,7 @@ async function openAlert(name){
box.hidden = false; document.getElementById("incidentTab").hidden = true;
document.getElementById("postureTab").hidden = true; document.getElementById("rulesTab").hidden = true;
document.getElementById("integrityTab").hidden = true;
document.getElementById("sidecarTab").hidden = true;
document.getElementById("eventsTab").hidden = true;
box.innerHTML='<div class="empty">loading snapshot</div>';
try{
@ -524,11 +541,13 @@ function renderTab(){
document.getElementById("alertsTab").hidden = state.tab !== "alerts";
document.getElementById("postureTab").hidden = state.tab !== "posture";
document.getElementById("integrityTab").hidden = state.tab !== "integrity";
document.getElementById("sidecarTab").hidden = state.tab !== "sidecar";
document.getElementById("rulesTab").hidden = state.tab !== "rules";
document.getElementById("eventsTab").hidden = state.tab !== "events";
if(state.tab === "incident") renderIncident();
if(state.tab === "posture") renderPosture();
if(state.tab === "integrity") renderIntegrity();
if(state.tab === "sidecar") renderSidecar();
if(state.tab === "rules") renderRules();
if(state.tab === "events") renderEvents();
}
@ -678,6 +697,76 @@ function renderIntegrity(){
box.append(grid);
}
function renderSidecar(){
// This is intentionally a read-only ledger, not a second incident workflow.
// Operators should not mistake validation-sidecar evidence for Python's
// production response plans, acknowledgements, or remediation authority.
const box = document.getElementById("sidecarTab");
const source = state.sidecar || {status:null, alerts:[], incidents:[], events:{events:[]}};
box.innerHTML = "";
if(source.error){
box.append(el("div","err","Go sidecar evidence is unavailable: "+source.error));
return;
}
const status = source.status || {};
if(!status.configured){
const empty = el("div","card");
empty.append(el("h2",null,"Go sidecar evidence"));
empty.append(el("div","fine","No migration evidence tree is configured. Set go_sidecar_state_dir to an isolated Go sidecar state directory; this console will only read its retained heartbeat, snapshots, incidents, and event log."));
box.append(empty);
return;
}
const summary = el("div","card");
summary.append(el("h2",null,"Go sidecar evidence"));
const heartbeat = status.heartbeat || {};
summary.append(tags([
"read-only", "state "+(status.available ? "available" : "unavailable"),
"heartbeat "+(heartbeat.status || "unknown"),
"alerts "+(source.alerts||[]).length, "incidents "+(source.incidents||[]).length
]));
summary.append(el("div","fine","Migration evidence is shown separately. Python alert, incident, and response-plan data remain authoritative."));
box.append(summary);
const grid = el("div","integrity-grid");
const health = el("article","state-card "+(heartbeat.status === "fresh" ? "ok" : "review"));
health.append(el("h3",null,"Heartbeat"));
addStateLine(health,"status",heartbeat.status || "unknown");
addStateLine(health,"age",humanAge(heartbeat.age_seconds));
addStateLine(health,"max age",humanAge(heartbeat.max_age_seconds));
addStateLine(health,"state dir",status.state_dir);
grid.append(health);
const evidence = el("article","state-card ok");
evidence.append(el("h3",null,"Retained evidence"));
(source.alerts||[]).slice(0,6).forEach(alert=>{
const row = el("div","path-row");
row.append(sev(alert.severity));
row.append(el("span",null,fmt(alert.time)+" — "+(alert.signatures||[]).join(", ")));
evidence.append(row);
});
if(!(source.alerts||[]).length) evidence.append(el("div","fine","No retained sidecar alert snapshots."));
grid.append(evidence);
const incidents = el("article","state-card ok");
incidents.append(el("h3",null,"Incident index"));
(source.incidents||[]).slice(0,6).forEach(incident=>{
const row = el("div","path-row");
row.append(sev(incident.severity));
row.append(el("span",null,incident.id+" — "+(incident.signatures||[]).join(", ")));
incidents.append(row);
});
if(!(source.incidents||[]).length) incidents.append(el("div","fine","No retained sidecar incidents."));
grid.append(incidents);
box.append(grid);
const events = el("div","card");
events.append(el("h3",null,"Retained sidecar events"));
const pre = el("pre");
const rows = source.events?.events || [];
pre.textContent = rows.length ? rows.map(row=>JSON.stringify(row)).join("\n") : "No retained sidecar events.";
events.append(pre);
box.append(events);
}
function renderRules(){
const box = document.getElementById("rulesTab");
const catalog = state.rules || {rules:[], by_event:{}, by_severity:{}, by_origin:{}};
@ -726,6 +815,7 @@ function showError(msg){
document.getElementById("incidentTab").innerHTML = '<div class="err">'+msg+'</div>';
document.getElementById("postureTab").innerHTML = '<div class="err">'+msg+'</div>';
document.getElementById("integrityTab").innerHTML = '<div class="err">'+msg+'</div>';
document.getElementById("sidecarTab").innerHTML = '<div class="err">'+msg+'</div>';
document.getElementById("rulesTab").innerHTML = '<div class="err">'+msg+'</div>';
document.getElementById("plan").innerHTML = '<div class="err">'+msg+'</div>';
}

View file

@ -175,6 +175,199 @@ def tail_events(cfg: Config, limit: int = 100) -> list[str]:
return lines[-limit:]
# --- isolated Go sidecar evidence (read-only migration consumer) ----------
def _go_sidecar_dir(cfg: Config) -> Path | None:
# This is an opt-in evidence boundary, not an alternate runtime root. The
# web process only derives paths beneath it and never creates or mutates it.
raw = cfg.go_sidecar_state_dir.strip()
return Path(raw) if raw else None
def go_sidecar_status(cfg: Config, now: float | None = None) -> dict:
"""Describe the separately configured Go evidence tree without touching it.
The Python daemon remains authoritative. This endpoint deliberately exposes
only whether the optional migration sidecar is configured and whether its
retained heartbeat is fresh; it never starts, stops, or probes the service.
"""
directory = _go_sidecar_dir(cfg)
report = {
"schema": schemas.GO_SIDECAR_V1,
"read_only": True,
"configured": directory is not None,
"state_dir": str(directory) if directory else "",
"available": bool(directory and directory.is_dir()),
"heartbeat": {"status": "not-configured"},
}
if directory is None:
return report
heartbeat = directory / "heartbeat"
try:
timestamp = int(heartbeat.read_text().strip())
except (OSError, ValueError):
report["heartbeat"] = {"status": "missing"}
return report
ts = time.time() if now is None else now
age = max(0.0, ts - timestamp)
report["heartbeat"] = {
"status": "fresh" if age <= cfg.heartbeat_max_age else "stale",
"timestamp": timestamp,
"age_seconds": age,
"max_age_seconds": cfg.heartbeat_max_age,
}
return report
def _go_snapshot_report(path: Path) -> dict | None:
# Sidecar files are retained forensic input, not trusted application data.
# Require the expected schema before presenting an arbitrary JSON document
# as a Sentinel alert snapshot.
try:
report = json.loads(path.read_text())
except (OSError, ValueError):
return None
if not isinstance(report, dict) or report.get("schema") != schemas.ALERT_SNAPSHOT_V1:
return None
return report
def list_go_sidecar_alerts(cfg: Config, limit: int = 200) -> list[dict]:
directory = _go_sidecar_dir(cfg)
if directory is None:
return []
alerts = []
for path in sorted(directory.glob("alert-*.json"), reverse=True)[:limit]:
report = _go_snapshot_report(path)
if report is None:
continue
records = report.get("alerts", [])
if not isinstance(records, list):
continue
alerts.append({
"name": path.with_suffix(".log").name,
"time": report.get("time", ""),
"host": report.get("host", ""),
"severity": report.get("severity", ""),
"signatures": sorted({a.get("signature", "") for a in records
if isinstance(a, dict)}),
"sids": sorted({a.get("sid", 0) for a in records
if isinstance(a, dict) and a.get("sid")}),
"count": len(records),
"source": "go-sidecar",
})
return alerts
def get_go_sidecar_alert(cfg: Config, name: str) -> dict | None:
if not isinstance(name, str) or not _ALERT_NAME.match(name):
return None
directory = _go_sidecar_dir(cfg)
if directory is None:
return None
base = directory / name
report = _go_snapshot_report(base.with_suffix(".json"))
if report is None:
return None
data = {"json": report, "source": "go-sidecar"}
try:
if base.is_file():
data["text"] = base.read_text()
except OSError:
pass
return data
def go_sidecar_events(cfg: Config, limit: int = 100) -> dict:
"""Return valid retained envelopes, failing closed if either segment corrupts."""
directory = _go_sidecar_dir(cfg)
result = {"events": [], "source": "go-sidecar", "error": None}
if directory is None:
return result
# Match the Go reader's chronology: the rotated segment precedes its active
# successor. Do not return a partial stream after corruption, because that
# could make an incomplete evidence chain look authoritative.
rows = []
for path in (directory / "events.jsonl.1", directory / "events.jsonl"):
try:
lines = path.read_text().splitlines()
except FileNotFoundError:
continue
except OSError as exc:
result["error"] = f"cannot read retained events: {exc}"
return result
for line in lines:
if not line:
continue
try:
row = json.loads(line)
except ValueError:
result["error"] = "retained event log is corrupt"
return result
if not isinstance(row, dict):
result["error"] = "retained event log contains a non-object record"
return result
rows.append(row)
result["events"] = rows[-max(0, limit):] if limit else []
return result
def list_go_sidecar_incidents(cfg: Config) -> list[dict]:
directory = _go_sidecar_dir(cfg)
if directory is None:
return []
try:
index = json.loads((directory / "incidents.json").read_text())
except (OSError, ValueError):
return []
if not isinstance(index, dict):
return []
# Atomic index replacement protects writers, but retained files can still
# be stale or manually damaged; schema filtering keeps this read-only view
# from interpreting unrelated JSON as an incident.
records = [item for item in index.values()
if isinstance(item, dict) and item.get("schema") == schemas.INCIDENT_V1]
def last_timestamp(item: dict) -> float:
try:
return float(item.get("last_ts", 0.0))
except (TypeError, ValueError):
return 0.0
return sorted(records, key=last_timestamp, reverse=True)
def get_go_sidecar_incident(cfg: Config, incident_id: str) -> dict | None:
incident = next((item for item in list_go_sidecar_incidents(cfg)
if item.get("id") == incident_id), None)
if incident is None:
return None
snapshots, timeline = [], []
for name in incident.get("snapshots", []):
if not isinstance(name, str):
timeline.append({"snapshot": str(name), "time": "?", "missing": True})
continue
alert = get_go_sidecar_alert(cfg, name)
if alert is None:
timeline.append({"snapshot": name, "time": "?", "missing": True})
continue
report = alert["json"]
snapshots.append(report)
timeline.append({
"snapshot": name,
"time": report.get("time", "?"),
"severity": report.get("severity", "?"),
"signatures": list(dict.fromkeys(
row.get("signature", "") for row in report.get("alerts", [])
if isinstance(row, dict))),
"pids": [row.get("pid") for row in report.get("processes", [])
if isinstance(row, dict) and isinstance(row.get("pid"), int)],
})
timeline.sort(key=lambda row: row.get("time", ""))
return {"schema": schemas.INCIDENT_VIEW_V1, "incident": incident,
"timeline": timeline, "snapshots": snapshots, "source": "go-sidecar"}
def daemon_status(cfg: Config) -> dict:
pid_alive = False
pidfile = cfg.log_dir / "sentinel.pid"
@ -505,6 +698,21 @@ class _Handler(BaseHTTPRequestHandler):
self._json_or_not_found(alert, send_body=send_body)
elif path == "/api/events":
self._json({"events": tail_events(cfg)}, send_body=send_body)
elif path == "/api/go-sidecar/status":
self._json(go_sidecar_status(cfg), send_body=send_body)
elif path == "/api/go-sidecar/alerts":
self._json(list_go_sidecar_alerts(cfg), send_body=send_body)
elif path.startswith("/api/go-sidecar/alerts/"):
alert = get_go_sidecar_alert(cfg, path.rsplit("/", 1)[-1])
self._json_or_not_found(alert, send_body=send_body)
elif path == "/api/go-sidecar/events":
self._json(go_sidecar_events(cfg), send_body=send_body)
elif path == "/api/go-sidecar/incidents":
self._json(list_go_sidecar_incidents(cfg), send_body=send_body)
elif path.startswith("/api/go-sidecar/incidents/"):
iid = unquote(path.rsplit("/", 1)[-1])
report = get_go_sidecar_incident(cfg, iid)
self._json_or_not_found(report, send_body=send_body)
elif path == "/api/posture":
self._json(posture_report(cfg), send_body=send_body)
elif path == "/api/rules":